[freyes,r=james-page] Ensure VIP is tied to a good mysqld instance.

Re-license charm as GPL-2 for compatibility with bundled ocf script.

.bzrignore

@@ -2,3 +2,6 @@ bin
 .coverage
 .pydevproject
 .project
+*.pyc
+*.pyo
+__pycache__

Makefile

@@ -9,6 +9,13 @@ lint:
 unit_test:
 	@$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests
 
+test:
+	@echo Starting amulet tests...
+	#NOTE(beisner): can remove -v after bug 1320357 is fixed
+	#   https://bugs.launchpad.net/amulet/+bug/1320357
+	# @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
+	echo "Tests disables; http://pad.lv/1446169"
+
 bin/charm_helpers_sync.py:
 	@mkdir -p bin
 	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \
@@ -16,6 +23,7 @@ bin/charm_helpers_sync.py:
 
 sync: bin/charm_helpers_sync.py
 	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml
+	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml
 
 publish: lint
 	bzr push lp:charms/trusty/percona-cluster

charm-helpers-tests.yaml

@@ -0,0 +1,5 @@
+branch: lp:charm-helpers
+destination: tests/charmhelpers
+include:
+    - contrib.amulet
+    - contrib.openstack.amulet

copyright

@@ -2,16 +2,28 @@ Format: http://dep.debian.net/deps/dep5/
 
 Files: *
 Copyright: Copyright 2011, Canonical Ltd., All Rights Reserved.
-License: GPL-3
+License: GPL-2
- This program is free software: you can redistribute it and/or modify
+
- it under the terms of the GNU General Public License as published by
+Files: ocf/percona/mysql_monitor
- the Free Software Foundation, either version 3 of the License, or
+Copyright: Copyright (c) 2013, Percona inc., Yves Trudeau, Michael Coburn
- (at your option) any later version.
+License: GPL-2
- .
+
- This program is distributed in the hope that it will be useful,
+License: GPL-2
- but WITHOUT ANY WARRANTY; without even the implied warranty of
+ This program is free software; you can redistribute it and/or modify
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ it under the terms of version 2 of the GNU General Public License as
- GNU General Public License for more details.
+ published by the Free Software Foundation.
- .
+
+ This program is distributed in the hope that it would be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+ Further, this software is distributed without any warranty that it is
+ free of the rightful claim of any third person regarding infringement
+ or the like.  Any license provided herein, whether implied or
+ otherwise, applies only to this software file.  Patent licenses, if
+ any, provided herein do not apply to combinations of this program with
+ other software, or any other product whatsoever.
+
  You should have received a copy of the GNU General Public License
- along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ along with this program; if not, write the Free Software Foundation,
+ Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.

hooks/percona_hooks.py

@@ -50,6 +50,7 @@ from percona_utils import (
     assert_charm_supports_ipv6,
     unit_sorted,
     get_db_helper,
+    install_mysql_ocf,
 )
 from charmhelpers.contrib.database.mysql import (
     PerconaClusterHelper,
@@ -72,6 +73,13 @@ from charmhelpers.contrib.network.ip import (
 hooks = Hooks()
 
 LEADER_RES = 'grp_percona_cluster'
+RES_MONITOR_PARAMS = ('params user="sstuser" password="%(sstpass)s" '
+                      'pid="/var/run/mysqld/mysqld.pid" '
+                      'socket="/var/run/mysqld/mysqld.sock" '
+                      'max_slave_lag="5" '
+                      'cluster_type="pxc" '
+                      'op monitor interval="1s" timeout="30s" '
+                      'OCF_CHECK_LEVEL="1"')
 
 
 @hooks.hook('install')
@@ -155,6 +163,13 @@ def config_changed():
         for unit in related_units(r_id):
             shared_db_changed(r_id, unit)
 
+    # (re)install pcmkr agent
+    install_mysql_ocf()
+
+    if relation_ids('ha'):
+        # make sure all the HA resources are (re)created
+        ha_relation_joined()
+
 
 @hooks.hook('cluster-relation-joined')
 def cluster_joined(relation_id=None):
@@ -387,17 +402,34 @@ def ha_relation_joined():
         vip_params = 'params ip="%s" cidr_netmask="%s" nic="%s"' % \
                      (vip, vip_cidr, vip_iface)
 
-    resources = {'res_mysql_vip': res_mysql_vip}
+    resources = {'res_mysql_vip': res_mysql_vip,
-    resource_params = {'res_mysql_vip': vip_params}
+                 'res_mysql_monitor': 'ocf:percona:mysql_monitor'}
+    db_helper = get_db_helper()
+    cfg_passwd = config('sst-password')
+    sstpsswd = db_helper.get_mysql_password(username='sstuser',
+                                            password=cfg_passwd)
+    resource_params = {'res_mysql_vip': vip_params,
+                       'res_mysql_monitor':
+                           RES_MONITOR_PARAMS % {'sstpass': sstpsswd}}
     groups = {'grp_percona_cluster': 'res_mysql_vip'}
 
+    clones = {'cl_mysql_monitor': 'res_mysql_monitor meta interleave=true'}
+
+    colocations = {'vip_mysqld': 'inf: grp_percona_cluster cl_mysql_monitor'}
+
+    locations = {'loc_percona_cluster':
+                 'grp_percona_cluster rule inf: writable eq 1'}
+
     for rel_id in relation_ids('ha'):
         relation_set(relation_id=rel_id,
                      corosync_bindiface=corosync_bindiface,
                      corosync_mcastport=corosync_mcastport,
                      resources=resources,
                      resource_params=resource_params,
-                     groups=groups)
+                     groups=groups,
+                     clones=clones,
+                     colocations=colocations,
+                     locations=locations)
 
 
 @hooks.hook('ha-relation-changed')

hooks/percona_utils.py

@@ -4,10 +4,12 @@ from subprocess import Popen, PIPE
 import socket
 import tempfile
 import os
+import shutil
 from charmhelpers.core.host import (
     lsb_release
 )
 from charmhelpers.core.hookenv import (
+    charm_dir,
     unit_get,
     relation_ids,
     related_units,
@@ -229,3 +231,18 @@ def unit_sorted(units):
     """Return a sorted list of unit names."""
     return sorted(
         units, lambda a, b: cmp(int(a.split('/')[-1]), int(b.split('/')[-1])))
+
+
+def install_mysql_ocf():
+    dest_dir = '/usr/lib/ocf/resource.d/percona/'
+    for fname in ['ocf/percona/mysql_monitor']:
+        src_file = os.path.join(charm_dir(), fname)
+        if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir)
+
+        dest_file = os.path.join(dest_dir, os.path.basename(src_file))
+        if not os.path.exists(dest_file):
+            log('Installing %s' % dest_file, level='INFO')
+            shutil.copy(src_file, dest_file)
+        else:
+            log("'%s' already exists, skipping" % dest_file, level='INFO')

ocf/percona/mysql_monitor

@@ -0,0 +1,636 @@
+#!/bin/bash
+#
+#
+#	MySQL_Monitor agent, set writeable and readable attributes based on the 
+#   state of the local MySQL, running and read_only or not.  The agent basis is
+#   the original "Dummy" agent written by Lars Marowsky-Brée and part of the
+#   Pacemaker distribution.  Many functions are from mysql_prm.
+#
+#
+# Copyright (c) 2013, Percona inc., Yves Trudeau, Michael Coburn
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of version 2 of the GNU General Public License as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it would be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# Further, this software is distributed without any warranty that it is
+# free of the rightful claim of any third person regarding infringement
+# or the like.  Any license provided herein, whether implied or
+# otherwise, applies only to this software file.  Patent licenses, if
+# any, provided herein do not apply to combinations of this program with
+# other software, or any other product whatsoever.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write the Free Software Foundation,
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
+#
+# Version: 20131119163921
+#
+# See usage() function below for more details...
+#
+# OCF instance parameters:
+#
+#  OCF_RESKEY_state
+#  OCF_RESKEY_user
+#  OCF_RESKEY_password
+#  OCF_RESKEY_client_binary
+#  OCF_RESKEY_pid
+#  OCF_RESKEY_socket
+#  OCF_RESKEY_reader_attribute
+#  OCF_RESKEY_reader_failcount
+#  OCF_RESKEY_writer_attribute
+#  OCF_RESKEY_max_slave_lag
+#  OCF_RESKEY_cluster_type
+#  
+#######################################################################
+# Initialization:
+
+: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat}
+. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs
+
+#######################################################################
+
+HOSTOS=`uname`
+if [ "X${HOSTOS}" = "XOpenBSD" ];then
+OCF_RESKEY_client_binary_default="/usr/local/bin/mysql"
+OCF_RESKEY_pid_default="/var/mysql/mysqld.pid"
+OCF_RESKEY_socket_default="/var/run/mysql/mysql.sock"
+else
+OCF_RESKEY_client_binary_default="/usr/bin/mysql"
+OCF_RESKEY_pid_default="/var/run/mysql/mysqld.pid"
+OCF_RESKEY_socket_default="/var/lib/mysql/mysql.sock"
+fi
+OCF_RESKEY_reader_attribute_default="readable"
+OCF_RESKEY_writer_attribute_default="writable"
+OCF_RESKEY_reader_failcount_default="1"
+OCF_RESKEY_user_default="root"
+OCF_RESKEY_password_default=""
+OCF_RESKEY_max_slave_lag_default="3600"
+OCF_RESKEY_cluster_type_default="replication"
+
+: ${OCF_RESKEY_state=${HA_RSCTMP}/mysql-monitor-${OCF_RESOURCE_INSTANCE}.state}
+: ${OCF_RESKEY_client_binary=${OCF_RESKEY_client_binary_default}}
+: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}}
+: ${OCF_RESKEY_socket=${OCF_RESKEY_socket_default}}
+: ${OCF_RESKEY_reader_attribute=${OCF_RESKEY_reader_attribute_default}}
+: ${OCF_RESKEY_reader_failcount=${OCF_RESKEY_reader_failcount_default}}
+: ${OCF_RESKEY_writer_attribute=${OCF_RESKEY_writer_attribute_default}}
+: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}}
+: ${OCF_RESKEY_password=${OCF_RESKEY_password_default}}
+: ${OCF_RESKEY_max_slave_lag=${OCF_RESKEY_max_slave_lag_default}}
+: ${OCF_RESKEY_cluster_type=${OCF_RESKEY_cluster_type_default}}
+
+MYSQL="$OCF_RESKEY_client_binary -A -S $OCF_RESKEY_socket --connect_timeout=10 --user=$OCF_RESKEY_user --password=$OCF_RESKEY_password "
+HOSTNAME=`uname -n`
+CRM_ATTR="${HA_SBIN_DIR}/crm_attribute -N $HOSTNAME "
+
+meta_data() {
+	cat <<END
+<?xml version="1.0"?>
+<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
+<resource-agent name="mysql_monitor" version="0.9">
+<version>1.0</version>
+
+<longdesc lang="en">
+This agent monitors the local MySQL instance and set the writable and readable
+attributes according to what it finds.  It checks if MySQL is running and if
+it is read-only or not.
+</longdesc>
+<shortdesc lang="en">Agent monitoring mysql</shortdesc>
+
+<parameters>
+<parameter name="state" unique="1">
+<longdesc lang="en">
+Location to store the resource state in.
+</longdesc>
+<shortdesc lang="en">State file</shortdesc>
+<content type="string" default="${HA_RSCTMP}/Mysql-monitor-${OCF_RESOURCE_INSTANCE}.state" />
+</parameter>
+
+<parameter name="user" unique="0">
+<longdesc lang="en">
+MySQL user to connect to the local MySQL instance to check the slave status and 
+if the read_only variable is set.  It requires the replication client priviledge.
+</longdesc>
+<shortdesc lang="en">MySQL user</shortdesc>
+<content type="string" default="${OCF_RESKEY_user_default}" />
+</parameter>
+
+<parameter name="password" unique="0">
+<longdesc lang="en">
+Password of the mysql user to connect to the local MySQL instance
+</longdesc>
+<shortdesc lang="en">MySQL password</shortdesc>
+<content type="string" default="${OCF_RESKEY_password_default}" />
+</parameter>
+
+<parameter name="client_binary" unique="0">
+<longdesc lang="en">
+MySQL Client Binary path.
+</longdesc>
+<shortdesc lang="en">MySQL client binary path</shortdesc>
+<content type="string" default="${OCF_RESKEY_client_binary_default}" />
+</parameter>
+
+<parameter name="socket" unique="0">
+<longdesc lang="en">
+Unix socket to use in order to connect to MySQL on the host
+</longdesc>
+<shortdesc lang="en">MySQL socket</shortdesc>
+<content type="string" default="${OCF_RESKEY_socket_default}" />
+</parameter>
+
+<parameter name="pid" unique="0">
+<longdesc lang="en">
+MySQL pid file, used to verify MySQL is running.
+</longdesc>
+<shortdesc lang="en">MySQL pid file</shortdesc>
+<content type="string" default="${OCF_RESKEY_pid_default}" />
+</parameter>
+
+<parameter name="reader_attribute" unique="0">
+<longdesc lang="en">
+The reader attribute in the cib that can be used by location rules to allow or not
+reader VIPs on a host.
+</longdesc>
+<shortdesc lang="en">Reader attribute</shortdesc>
+<content type="string" default="${OCF_RESKEY_reader_attribute_default}" />
+</parameter>
+
+<parameter name="writer_attribute" unique="0">
+<longdesc lang="en">
+The reader attribute in the cib that can be used by location rules to allow or not
+reader VIPs on a host.
+</longdesc>
+<shortdesc lang="en">Writer attribute</shortdesc>
+<content type="string" default="${OCF_RESKEY_writer_attribute_default}" />
+</parameter>
+
+<parameter name="max_slave_lag" unique="0" required="0">
+<longdesc lang="en">
+The maximum number of seconds a replication slave is allowed to lag
+behind its master in order to have a reader VIP on it.
+</longdesc>
+<shortdesc lang="en">Maximum time (seconds) a MySQL slave is allowed
+to lag behind a master</shortdesc>
+<content type="integer" default="${OCF_RESKEY_max_slave_lag_default}"/>
+</parameter>
+
+<parameter name="cluster_type" unique="0" required="0">
+<longdesc lang="en">
+Type of cluster, three possible values: pxc, replication, read-only.  "pxc" is 
+for Percona XtraDB cluster, it uses the clustercheck script and set the 
+reader_attribute and writer_attribute according to the return code.  
+"replication" checks the read-only state and the slave status, only writable 
+node(s) will get the writer_attribute (and the reader_attribute) and on the 
+read-only nodes, replication status will be checked and the reader_attribute set 
+according to the state.  "read-only" will just check if the read-only variable, 
+if read/write, it will get both the writer_attribute and reader_attribute set, if
+read-only it will get only the reader_attribute.
+</longdesc>
+<shortdesc lang="en">Type of cluster</shortdesc>
+<content type="string" default="${OCF_RESKEY_cluster_type_default}"/>
+</parameter>
+
+</parameters>
+
+<actions>
+<action name="start"        timeout="20" />
+<action name="stop"         timeout="20" />
+<action name="monitor"      timeout="20" interval="10" depth="0" />
+<action name="reload"       timeout="20" />
+<action name="migrate_to"   timeout="20" />
+<action name="migrate_from" timeout="20" />
+<action name="meta-data"    timeout="5" />
+<action name="validate-all"   timeout="20" />
+</actions>
+</resource-agent>
+END
+}
+
+#######################################################################
+# Non API functions
+
+# Extract fields from slave status
+parse_slave_info() {
+    # Extracts field $1 from result of "SHOW SLAVE STATUS\G" from file $2
+    sed -ne "s/^.* $1: \(.*\)$/\1/p" < $2
+}
+
+# Read the slave status and 
+get_slave_info() {
+
+    local mysql_options tmpfile
+            
+    if [ "$master_log_file" -a "$master_host" ]; then
+        # variables are already defined, get_slave_info has been run before
+        return $OCF_SUCCESS
+    else
+        tmpfile=`mktemp ${HA_RSCTMP}/check_slave.${OCF_RESOURCE_INSTANCE}.XXXXXX`
+
+        mysql_run -Q -sw -O $MYSQL $MYSQL_OPTIONS_REPL \
+        -e 'SHOW SLAVE STATUS\G' > $tmpfile
+
+        if [ -s $tmpfile ]; then
+            master_host=`parse_slave_info Master_Host $tmpfile`
+            slave_sql=`parse_slave_info Slave_SQL_Running $tmpfile`
+            slave_io=`parse_slave_info Slave_IO_Running $tmpfile`
+            slave_io_state=`parse_slave_info Slave_IO_State $tmpfile`
+            last_errno=`parse_slave_info Last_Errno $tmpfile`
+            secs_behind=`parse_slave_info Seconds_Behind_Master $tmpfile`
+            ocf_log debug "MySQL instance has a non empty slave status"
+        else
+            # Instance produced an empty "SHOW SLAVE STATUS" output --
+            # instance is not a slave
+            
+            ocf_log err "check_slave invoked on an instance that is not a replication slave."
+	        rm -f $tmpfile
+            return $OCF_ERR_GENERIC
+        fi
+        rm -f $tmpfile
+        return $OCF_SUCCESS
+    fi
+}
+
+get_read_only() {
+    # Check if read-only is set
+    local read_only_state
+
+    read_only_state=`mysql_run -Q -sw -O $MYSQL -N $MYSQL_OPTIONS_REPL \
+        -e "SHOW VARIABLES like 'read_only'" | awk '{print $2}'`
+
+    if [ "$read_only_state" = "ON" ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# get the attribute controlling the readers VIP
+get_reader_attr() {
+    local attr_value
+    local rc
+
+    attr_value=`$CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} --query -q`
+    rc=$?
+    if [ "$rc" -eq "0" ]; then
+        echo $attr_value
+    else
+        echo -1
+    fi
+
+}
+
+# Set the attribute controlling the readers VIP
+set_reader_attr() {
+    local curr_attr_value
+
+    curr_attr_value=$(get_reader_attr)
+    
+    if [ "$1" -eq "0" ]; then
+        if [ "$curr_attr_value" -gt "0" ]; then
+            curr_attr_value=$((${curr_attr_value}-1))    
+            $CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} -v $curr_attr_value
+        else
+            $CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} -v 0
+        fi
+    else
+        if [ "$curr_attr_value" -ne "$OCF_RESKEY_reader_failcount" ]; then
+            $CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} -v $OCF_RESKEY_reader_failcount
+        fi
+    fi
+
+}
+
+# get the attribute controlling the writer VIP
+get_writer_attr() {
+    local attr_value
+    local rc
+
+    attr_value=`$CRM_ATTR -l reboot --name ${OCF_RESKEY_writer_attribute} --query -q`
+    rc=$?
+    if [ "$rc" -eq "0" ]; then
+        echo $attr_value
+    else
+        echo -1
+    fi
+
+}
+
+# Set the attribute controlling the writer VIP
+set_writer_attr() {
+    local curr_attr_value
+
+    curr_attr_value=$(get_writer_attr)
+    
+    if [ "$1" -ne "$curr_attr_value" ]; then
+        if [ "$1" -eq "0" ]; then
+            $CRM_ATTR -l reboot --name ${OCF_RESKEY_writer_attribute} -v 0
+        else
+            $CRM_ATTR -l reboot --name ${OCF_RESKEY_writer_attribute} -v 1
+        fi
+    fi
+}
+
+#
+# mysql_run: Run a mysql command, log its output and return the proper error code.
+# Usage:   mysql_run [-Q] [-info|-warn|-err] [-O] [-sw] <command>
+#       -Q: don't log the output of the command if it succeeds
+#       -info|-warn|-err: log the output of the command at given
+#               severity if it fails (defaults to err)
+#       -O: echo the output of the command
+#       -sw: Suppress 5.6 client warning when password is used on the command line
+# Adapted from ocf_run.
+#
+mysql_run() {
+        local rc
+        local output outputfile
+        local verbose=1
+        local returnoutput
+        local loglevel=err
+        local suppress_56_password_warning
+        local var
+
+        for var in 1 2 3 4
+        do
+            case "$1" in
+                "-Q")
+                    verbose=""
+                    shift 1;;
+                "-info"|"-warn"|"-err")
+                    loglevel=`echo $1 | sed -e s/-//g`
+                    shift 1;;
+                "-O")
+                    returnoutput=1
+                    shift 1;;
+                "-sw")
+                    suppress_56_password_warning=1
+                    shift 1;;
+
+                *)
+                    ;;
+            esac
+        done
+
+        outputfile=`mktemp ${HA_RSCTMP}/mysql_run.${OCF_RESOURCE_INSTANCE}.XXXXXX`
+        error=`"$@" 2>&1 1>$outputfile`
+        rc=$?
+        if [ "$suppress_56_password_warning" -eq 1 ]; then
+                error=`echo "$error" | egrep -v '^Warning: Using a password on the command line'`
+        fi
+        output=`cat $outputfile`
+        rm -f $outputfile
+
+        if [ $rc -eq 0 ]; then
+            if [ "$verbose" -a ! -z "$output" ]; then
+                ocf_log info "$output"
+            fi
+
+            if [ "$returnoutput" -a ! -z "$output" ]; then
+                echo "$output"
+            fi
+
+            MYSQL_LAST_ERR=$OCF_SUCCESS
+            return $OCF_SUCCESS
+        else
+            if [ ! -z "$error" ]; then                 
+                ocf_log $loglevel "$error"
+                regex='^ERROR ([[:digit:]]{4}).*'
+                if [[ $error =~ $regex ]]; then
+                    mysql_code=${BASH_REMATCH[1]}
+                    if [ -n "$mysql_code" ]; then
+                        MYSQL_LAST_ERR=$mysql_code
+                        return $rc
+                    fi
+                fi
+            else
+                ocf_log $loglevel "command failed: $*"
+            fi
+            # No output to parse so return the standard exit code.
+            MYSQL_LAST_ERR=$rc
+            return $rc
+        fi
+}
+
+
+
+
+#######################################################################
+# API functions
+
+mysql_monitor_usage() {
+	cat <<END
+usage: $0 {start|stop|monitor|migrate_to|migrate_from|validate-all|meta-data}
+
+Expects to have a fully populated OCF RA-compliant environment set.
+END
+}
+
+mysql_monitor_start() {
+    
+    # Initialise the attribute in the cib if they are not already there.
+    if [ $(get_reader_attr) -eq -1 ]; then
+        set_reader_attr 0
+    fi
+    
+    if [ $(get_writer_attr) -eq -1 ]; then
+        set_writer_attr 0
+    fi
+    
+    mysql_monitor
+    mysql_monitor_monitor
+    if [ $? =  $OCF_SUCCESS ]; then
+	return $OCF_SUCCESS
+    fi
+    touch ${OCF_RESKEY_state}
+}
+
+mysql_monitor_stop() {
+    
+    set_reader_attr 0
+    set_writer_attr 0
+    
+    mysql_monitor_monitor
+    if [ $? =  $OCF_SUCCESS ]; then
+	rm ${OCF_RESKEY_state}
+    fi
+    return $OCF_SUCCESS
+
+}
+
+# Monitor MySQL, not the agent itself
+mysql_monitor() {
+    if [  -e $OCF_RESKEY_pid ]; then
+        pid=`cat $OCF_RESKEY_pid`;
+        if [ -d /proc -a -d /proc/1 ]; then
+            [ "u$pid" != "u" -a -d /proc/$pid ]
+        else
+            kill -s 0 $pid >/dev/null 2>&1
+        fi
+
+        if [ $? -eq 0 ]; then
+
+            case ${OCF_RESKEY_cluster_type} in
+            'replication'|'REPLICATION')
+                if get_read_only; then
+                    # a slave?
+                    
+                    set_writer_attr 0
+                    
+                    get_slave_info
+                    rc=$?
+
+                    if [ $rc -eq 0 ]; then
+                       # show slave status is not empty
+                       # Is there a master_log_file defined?  (master_log_file is deleted 
+                       # by reset slave
+                       if [ "$master_log_file" ]; then
+                          # is read_only but no slave config...
+                          
+                          set_reader_attr 0
+                          
+                       else
+                          # has a slave config
+                          
+                          if [ "$slave_sql" = 'Yes' -a "$slave_io" = 'Yes' ]; then
+                              # $secs_behind can be NULL so must be tested only
+                              # if replication is OK
+                              if [ $secs_behind -gt $OCF_RESKEY_max_slave_lag ]; then
+                                  set_reader_attr 0
+                              else
+                                  set_reader_attr 1
+                              fi
+                          else
+                              set_reader_attr 0
+                          fi
+                       fi
+                    else
+                       # "SHOW SLAVE STATUS" returns an empty set if instance is not a
+                       # replication slave
+                       
+                       set_reader_attr 0
+                       
+                    fi
+                else
+                    # host is RW
+                    set_reader_attr 1
+                    set_writer_attr 1
+                fi
+                ;;
+                
+            'pxc'|'PXC')
+                pxcstat=`/usr/bin/clustercheck $OCF_RESKEY_user $OCF_RESKEY_password `
+                if [ $? -eq 0 ]; then
+                    set_reader_attr 1
+                    set_writer_attr 1
+                else
+                    set_reader_attr 0
+                    set_writer_attr 0
+                fi
+                
+                ;;
+                
+            'read-only'|'READ-ONLY')
+                if get_read_only; then
+                    set_reader_attr 1
+                    set_writer_attr 0                    
+                else
+                    set_reader_attr 1
+                    set_writer_attr 1
+                fi
+                ;;
+                
+            esac
+        else
+            ocf_log $1 "MySQL is not running, but there is a pidfile"
+            set_reader_attr 0
+            set_writer_attr 0
+        fi
+    else
+        ocf_log $1 "MySQL is not running"
+        set_reader_attr 0
+        set_writer_attr 0
+    fi
+}
+    
+mysql_monitor_monitor() {
+	# Monitor _MUST!_ differentiate correctly between running
+	# (SUCCESS), failed (ERROR) or _cleanly_ stopped (NOT RUNNING).
+	# That is THREE states, not just yes/no.
+    
+	if [ -f ${OCF_RESKEY_state} ]; then
+	    return $OCF_SUCCESS
+	fi
+	if false ; then
+		return $OCF_ERR_GENERIC
+	fi
+	return $OCF_NOT_RUNNING
+}
+
+mysql_monitor_validate() {
+    
+    # Is the state directory writable? 
+    state_dir=`dirname "$OCF_RESKEY_state"`
+    touch "$state_dir/$$"
+    if [ $? != 0 ]; then
+	return $OCF_ERR_ARGS
+    fi
+    rm "$state_dir/$$"
+
+    return $OCF_SUCCESS
+}
+
+##########################################################################
+# If DEBUG_LOG is set, make this resource agent easy to debug: set up the
+# debug log and direct all output to it.  Otherwise, redirect to /dev/null.
+# The log directory must be a directory owned by root, with permissions 0700,
+# and the log must be writable and not a symlink.
+##########################################################################
+DEBUG_LOG="/tmp/mysql_monitor.ocf.ra.debug/log"
+if [ "${DEBUG_LOG}" -a -w "${DEBUG_LOG}" -a ! -L "${DEBUG_LOG}" ]; then
+    DEBUG_LOG_DIR="${DEBUG_LOG%/*}"
+    if [ -d "${DEBUG_LOG_DIR}" ]; then
+        exec 9>>"$DEBUG_LOG"
+        exec 2>&9
+        date >&9
+        echo "$*" >&9
+        env | grep OCF_ | sort >&9
+        set -x
+    else
+        exec 9>/dev/null
+    fi
+fi
+
+
+case $__OCF_ACTION in
+meta-data)	meta_data
+		exit $OCF_SUCCESS
+		;;
+start)		mysql_monitor_start;;
+stop)		mysql_monitor_stop;;
+monitor)	mysql_monitor
+            mysql_monitor_monitor;;
+migrate_to)	ocf_log info "Migrating ${OCF_RESOURCE_INSTANCE} to ${OCF_RESKEY_CRM_meta_migrate_target}."
+	        mysql_monitor_stop
+		;;
+migrate_from)	ocf_log info "Migrating ${OCF_RESOURCE_INSTANCE} from ${OCF_RESKEY_CRM_meta_migrate_source}."
+	        mysql_monitor_start
+		;;
+reload)		ocf_log info "Reloading ${OCF_RESOURCE_INSTANCE} ..."
+		;;
+validate-all)	mysql_monitor_validate;;
+usage|help)	mysql_monitor_usage
+		exit $OCF_SUCCESS
+		;;
+*)		mysql_monitor_usage
+		exit $OCF_ERR_UNIMPLEMENTED
+		;;
+esac
+rc=$?
+ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc"
+exit $rc
+

setup.cfg

@@ -0,0 +1,6 @@
+[nosetests]
+verbosity=2
+with-coverage=1
+cover-erase=1
+cover-package=hooks
+

templates/my.cnf

@@ -11,6 +11,7 @@ wsrep_provider_options = {{ wsrep_provider_options }}
 
 datadir=/var/lib/mysql
 user=mysql
+pid_file = /var/run/mysqld/mysqld.pid
 
 # Path to Galera library
 wsrep_provider=/usr/lib/libgalera_smm.so

tests/00-setup.sh

@@ -0,0 +1,29 @@
+#!/bin/bash -x
+# The script installs amulet and other tools needed for the amulet tests.
+
+# Get the status of the amulet package, this returns 0 of package is installed.
+dpkg -s amulet
+if [ $? -ne 0 ]; then
+  # Install the Amulet testing harness.
+  sudo add-apt-repository -y ppa:juju/stable
+  sudo apt-get update 
+  sudo apt-get install -y -q amulet juju-core charm-tools
+fi
+
+
+PACKAGES="python3 python3-yaml"
+for pkg in $PACKAGES; do
+  dpkg -s python3
+  if [ $? -ne 0 ]; then
+    sudo apt-get install -y -q $pkg
+  fi
+done
+
+
+#if [ ! -f "$(dirname $0)/../local.yaml" ]; then
+#  echo "To run these amulet tests a vip is needed, create a file called \
+#local.yaml in the charm dir, this file must contain a 'vip', if you're \
+#using the local provider with lxc you could use a free IP from the range \
+#10.0.3.0/24"
+#  exit 1
+#fi

tests/10-deploy_test.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python3
+# test percona-cluster (3 nodes)
+
+import basic_deployment
+import time
+
+
+class ThreeNode(basic_deployment.BasicDeployment):
+    def __init__(self):
+        super(ThreeNode, self).__init__(units=3)
+
+    def run(self):
+        super(ThreeNode, self).run()
+        # we are going to kill the master
+        old_master = self.master_unit
+        self.master_unit.run('sudo poweroff')
+
+        time.sleep(10)  # give some time to pacemaker to react
+        new_master = self.find_master()
+        assert new_master is not None, "master unit not found"
+        assert (new_master.info['public-address'] !=
+                    old_master.info['public-address'])
+
+        assert self.is_port_open(address=self.vip), 'cannot connect to vip'
+
+
+if __name__ == "__main__":
+    t = ThreeNode()
+    t.run()

tests/20-broken-mysqld.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python3
+# test percona-cluster (3 nodes)
+
+import basic_deployment
+import time
+
+
+class ThreeNode(basic_deployment.BasicDeployment):
+    def __init__(self):
+        super(ThreeNode, self).__init__(units=3)
+
+    def run(self):
+        super(ThreeNode, self).run()
+        # we are going to kill the master
+        old_master = self.master_unit
+        print('stopping mysql in %s' % str(self.master_unit.info))
+        self.master_unit.run('sudo service mysql stop')
+
+        print('looking for the new master')
+        i = 0
+        changed = False
+        while i < 10 and not changed:
+            i += 1
+            time.sleep(5)  # give some time to pacemaker to react
+            new_master = self.find_master()
+
+            if (new_master and new_master.info['unit_name'] !=
+                    old_master.info['unit_name']):
+                changed = True
+
+        assert changed, "The master didn't change"
+
+        assert self.is_port_open(address=self.vip), 'cannot connect to vip'
+
+
+if __name__ == "__main__":
+    t = ThreeNode()
+    t.run()

tests/30-kill-9-mysqld.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python3
+# test percona-cluster (3 nodes)
+
+import basic_deployment
+import time
+
+
+class ThreeNode(basic_deployment.BasicDeployment):
+    def __init__(self):
+        super(ThreeNode, self).__init__(units=3)
+
+    def run(self):
+        super(ThreeNode, self).run()
+        # we are going to kill the master
+        old_master = self.master_unit
+        print('kill-9 mysqld in %s' % str(self.master_unit.info))
+        self.master_unit.run('sudo killall -9 mysqld')
+
+        print('looking for the new master')
+        i = 0
+        changed = False
+        while i < 10 and not changed:
+            i += 1
+            time.sleep(5)  # give some time to pacemaker to react
+            new_master = self.find_master()
+
+            if (new_master and new_master.info['unit_name'] !=
+                    old_master.info['unit_name']):
+                changed = True
+
+        assert changed, "The master didn't change"
+
+        assert self.is_port_open(address=self.vip), 'cannot connect to vip'
+
+
+if __name__ == "__main__":
+    t = ThreeNode()
+    t.run()

tests/basic_deployment.py

@@ -0,0 +1,151 @@
+import amulet
+import os
+import time
+import telnetlib
+import unittest
+import yaml
+from charmhelpers.contrib.openstack.amulet.deployment import (
+    OpenStackAmuletDeployment
+)
+
+
+class BasicDeployment(OpenStackAmuletDeployment):
+    def __init__(self, vip=None, units=1, series="trusty", openstack=None,
+                 source=None, stable=False):
+        super(BasicDeployment, self).__init__(series, openstack, source,
+                                              stable)
+        self.units = units
+        self.master_unit = None
+        self.vip = None
+        if vip:
+            self.vip = vip
+        elif 'AMULET_OS_VIP' in os.environ:
+            self.vip = os.environ.get('AMULET_OS_VIP')
+        elif os.path.isfile('local.yaml'):
+            with open('local.yaml', 'rb') as f:
+                self.cfg = yaml.safe_load(f.read())
+
+            self.vip = self.cfg.get('vip')
+        else:
+            amulet.raise_status(amulet.SKIP,
+                                ("please set the vip in local.yaml or env var "
+                                 "AMULET_OS_VIP to run this test suite"))
+
+    def _add_services(self):
+        """Add services
+
+           Add the services that we're testing, where percona-cluster is local,
+           and the rest of the service are from lp branches that are
+           compatible with the local charm (e.g. stable or next).
+           """
+        this_service = {'name': 'percona-cluster',
+                        'units': self.units}
+        other_services = [{'name': 'hacluster'}]
+        super(BasicDeployment, self)._add_services(this_service,
+                                                   other_services)
+
+    def _add_relations(self):
+        """Add all of the relations for the services."""
+        relations = {'percona-cluster:ha': 'hacluster:ha'}
+        super(BasicDeployment, self)._add_relations(relations)
+
+    def _configure_services(self):
+        """Configure all of the services."""
+        cfg_percona = {'sst-password': 'ubuntu',
+                       'root-password': 't00r',
+                       'dataset-size': '512M',
+                       'vip': self.vip}
+
+        cfg_ha = {'debug': True,
+                  'corosync_mcastaddr': '226.94.1.4',
+                  'corosync_key': ('xZP7GDWV0e8Qs0GxWThXirNNYlScgi3sRTdZk/IXKD'
+                                   'qkNFcwdCWfRQnqrHU/6mb6sz6OIoZzX2MtfMQIDcXu'
+                                   'PqQyvKuv7YbRyGHmQwAWDUA4ed759VWAO39kHkfWp9'
+                                   'y5RRk/wcHakTcWYMwm70upDGJEP00YT3xem3NQy27A'
+                                   'C1w=')}
+
+        configs = {'percona-cluster': cfg_percona,
+                   'hacluster': cfg_ha}
+        super(BasicDeployment, self)._configure_services(configs)
+
+    def run(self):
+        # The number of seconds to wait for the environment to setup.
+        seconds = 1200
+
+        self._add_services()
+        self._add_relations()
+        self._configure_services()
+        self._deploy()
+
+        i = 0
+        while i < 30 and not self.master_unit:
+            self.master_unit = self.find_master()
+            i += 1
+            time.sleep(10)
+
+        assert self.master_unit is not None, 'percona-cluster vip not found'
+
+        output, code = self.master_unit.run('sudo crm_verify --live-check')
+        assert code == 0, "'crm_verify --live-check' failed"
+
+        resources = ['res_mysql_vip']
+        resources += ['res_mysql_monitor:%d' % i for i in range(self.units)]
+
+        assert sorted(self.get_pcmkr_resources()) == sorted(resources)
+
+        for i in range(self.units):
+            uid = 'percona-cluster/%d' % i
+            unit = self.d.sentry.unit[uid]
+            assert self.is_mysqld_running(unit), 'mysql not running: %s' % uid
+
+    def find_master(self):
+        for unit_id, unit in self.d.sentry.unit.items():
+            if not unit_id.startswith('percona-cluster/'):
+                continue
+
+            # is the vip running here?
+            output, code = unit.run('sudo ip a | grep "inet %s/"' % self.vip)
+            print('---')
+            print(unit_id)
+            print(output)
+            if code == 0:
+                print('vip(%s) running in %s' % (self.vip, unit_id))
+                return unit
+
+    def get_pcmkr_resources(self, unit=None):
+        if unit:
+            u = unit
+        else:
+            u = self.master_unit
+
+        output, code = u.run('sudo crm_resource -l')
+
+        assert code == 0, 'could not get "crm resource list"'
+
+        return output.split('\n')
+
+    def is_mysqld_running(self, unit=None):
+        if unit:
+            u = unit
+        else:
+            u = self.master_unit
+
+        output, code = u.run('pidof mysqld')
+
+        if code != 0:
+            return False
+
+        return self.is_port_open(u, '3306')
+
+    def is_port_open(self, unit=None, port='3306', address=None):
+        if unit:
+            addr = unit.info['public-address']
+        elif address:
+            addr = address
+        else:
+            raise Exception('Please provide a unit or address')
+        try:
+            telnetlib.Telnet(addr, port)
+            return True
+        except TimeoutError:  # noqa this exception only available in py3
+            return False

tests/charmhelpers/__init__.py

@@ -0,0 +1,38 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+# Bootstrap charm-helpers, installing its dependencies if necessary using
+# only standard libraries.
+import subprocess
+import sys
+
+try:
+    import six  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-six'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-six'])
+    import six  # flake8: noqa
+
+try:
+    import yaml  # flake8: noqa
+except ImportError:
+    if sys.version_info.major == 2:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml'])
+    else:
+        subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml'])
+    import yaml  # flake8: noqa

tests/charmhelpers/contrib/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

tests/charmhelpers/contrib/amulet/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

tests/charmhelpers/contrib/amulet/deployment.py

@@ -0,0 +1,93 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import amulet
+import os
+import six
+
+
+class AmuletDeployment(object):
+    """Amulet deployment.
+
+       This class provides generic Amulet deployment and test runner
+       methods.
+       """
+
+    def __init__(self, series=None):
+        """Initialize the deployment environment."""
+        self.series = None
+
+        if series:
+            self.series = series
+            self.d = amulet.Deployment(series=self.series)
+        else:
+            self.d = amulet.Deployment()
+
+    def _add_services(self, this_service, other_services):
+        """Add services.
+
+           Add services to the deployment where this_service is the local charm
+           that we're testing and other_services are the other services that
+           are being used in the local amulet tests.
+           """
+        if this_service['name'] != os.path.basename(os.getcwd()):
+            s = this_service['name']
+            msg = "The charm's root directory name needs to be {}".format(s)
+            amulet.raise_status(amulet.FAIL, msg=msg)
+
+        if 'units' not in this_service:
+            this_service['units'] = 1
+
+        self.d.add(this_service['name'], units=this_service['units'])
+
+        for svc in other_services:
+            if 'location' in svc:
+                branch_location = svc['location']
+            elif self.series:
+                branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
+            else:
+                branch_location = None
+
+            if 'units' not in svc:
+                svc['units'] = 1
+
+            self.d.add(svc['name'], charm=branch_location, units=svc['units'])
+
+    def _add_relations(self, relations):
+        """Add all of the relations for the services."""
+        for k, v in six.iteritems(relations):
+            self.d.relate(k, v)
+
+    def _configure_services(self, configs):
+        """Configure all of the services."""
+        for service, config in six.iteritems(configs):
+            self.d.configure(service, config)
+
+    def _deploy(self):
+        """Deploy environment and wait for all hooks to finish executing."""
+        try:
+            self.d.setup(timeout=900)
+            self.d.sentry.wait(timeout=900)
+        except amulet.helpers.TimeoutError:
+            amulet.raise_status(amulet.FAIL, msg="Deployment timed out")
+        except Exception:
+            raise
+
+    def run_tests(self):
+        """Run all of the methods that are prefixed with 'test_'."""
+        for test in dir(self):
+            if test.startswith('test_'):
+                getattr(self, test)()

tests/charmhelpers/contrib/amulet/utils.py

@@ -0,0 +1,316 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import ConfigParser
+import io
+import logging
+import re
+import sys
+import time
+
+import six
+
+
+class AmuletUtils(object):
+    """Amulet utilities.
+
+       This class provides common utility functions that are used by Amulet
+       tests.
+       """
+
+    def __init__(self, log_level=logging.ERROR):
+        self.log = self.get_logger(level=log_level)
+
+    def get_logger(self, name="amulet-logger", level=logging.DEBUG):
+        """Get a logger object that will log to stdout."""
+        log = logging
+        logger = log.getLogger(name)
+        fmt = log.Formatter("%(asctime)s %(funcName)s "
+                            "%(levelname)s: %(message)s")
+
+        handler = log.StreamHandler(stream=sys.stdout)
+        handler.setLevel(level)
+        handler.setFormatter(fmt)
+
+        logger.addHandler(handler)
+        logger.setLevel(level)
+
+        return logger
+
+    def valid_ip(self, ip):
+        if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
+            return True
+        else:
+            return False
+
+    def valid_url(self, url):
+        p = re.compile(
+            r'^(?:http|ftp)s?://'
+            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # noqa
+            r'localhost|'
+            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
+            r'(?::\d+)?'
+            r'(?:/?|[/?]\S+)$',
+            re.IGNORECASE)
+        if p.match(url):
+            return True
+        else:
+            return False
+
+    def validate_services(self, commands):
+        """Validate services.
+
+           Verify the specified services are running on the corresponding
+           service units.
+           """
+        for k, v in six.iteritems(commands):
+            for cmd in v:
+                output, code = k.run(cmd)
+                if code != 0:
+                    return "command `{}` returned {}".format(cmd, str(code))
+        return None
+
+    def _get_config(self, unit, filename):
+        """Get a ConfigParser object for parsing a unit's config file."""
+        file_contents = unit.file_contents(filename)
+        config = ConfigParser.ConfigParser()
+        config.readfp(io.StringIO(file_contents))
+        return config
+
+    def validate_config_data(self, sentry_unit, config_file, section,
+                             expected):
+        """Validate config file data.
+
+           Verify that the specified section of the config file contains
+           the expected option key:value pairs.
+           """
+        config = self._get_config(sentry_unit, config_file)
+
+        if section != 'DEFAULT' and not config.has_section(section):
+            return "section [{}] does not exist".format(section)
+
+        for k in expected.keys():
+            if not config.has_option(section, k):
+                return "section [{}] is missing option {}".format(section, k)
+            if config.get(section, k) != expected[k]:
+                return "section [{}] {}:{} != expected {}:{}".format(
+                       section, k, config.get(section, k), k, expected[k])
+        return None
+
+    def _validate_dict_data(self, expected, actual):
+        """Validate dictionary data.
+
+           Compare expected dictionary data vs actual dictionary data.
+           The values in the 'expected' dictionary can be strings, bools, ints,
+           longs, or can be a function that evaluate a variable and returns a
+           bool.
+           """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        self.log.debug('expected: {}'.format(repr(expected)))
+
+        for k, v in six.iteritems(expected):
+            if k in actual:
+                if (isinstance(v, six.string_types) or
+                        isinstance(v, bool) or
+                        isinstance(v, six.integer_types)):
+                    if v != actual[k]:
+                        return "{}:{}".format(k, actual[k])
+                elif not v(actual[k]):
+                    return "{}:{}".format(k, actual[k])
+            else:
+                return "key '{}' does not exist".format(k)
+        return None
+
+    def validate_relation_data(self, sentry_unit, relation, expected):
+        """Validate actual relation data based on expected relation data."""
+        actual = sentry_unit.relation(relation[0], relation[1])
+        return self._validate_dict_data(expected, actual)
+
+    def _validate_list_data(self, expected, actual):
+        """Compare expected list vs actual list data."""
+        for e in expected:
+            if e not in actual:
+                return "expected item {} not found in actual list".format(e)
+        return None
+
+    def not_null(self, string):
+        if string is not None:
+            return True
+        else:
+            return False
+
+    def _get_file_mtime(self, sentry_unit, filename):
+        """Get last modification time of file."""
+        return sentry_unit.file_stat(filename)['mtime']
+
+    def _get_dir_mtime(self, sentry_unit, directory):
+        """Get last modification time of directory."""
+        return sentry_unit.directory_stat(directory)['mtime']
+
+    def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
+        """Get process' start time.
+
+           Determine start time of the process based on the last modification
+           time of the /proc/pid directory. If pgrep_full is True, the process
+           name is matched against the full command line.
+           """
+        if pgrep_full:
+            cmd = 'pgrep -o -f {}'.format(service)
+        else:
+            cmd = 'pgrep -o {}'.format(service)
+        cmd = cmd + '  | grep  -v pgrep || exit 0'
+        cmd_out = sentry_unit.run(cmd)
+        self.log.debug('CMDout: ' + str(cmd_out))
+        if cmd_out[0]:
+            self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
+            proc_dir = '/proc/{}'.format(cmd_out[0].strip())
+            return self._get_dir_mtime(sentry_unit, proc_dir)
+
+    def service_restarted(self, sentry_unit, service, filename,
+                          pgrep_full=False, sleep_time=20):
+        """Check if service was restarted.
+
+           Compare a service's start time vs a file's last modification time
+           (such as a config file for that service) to determine if the service
+           has been restarted.
+           """
+        time.sleep(sleep_time)
+        if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
+                self._get_file_mtime(sentry_unit, filename)):
+            return True
+        else:
+            return False
+
+    def service_restarted_since(self, sentry_unit, mtime, service,
+                                pgrep_full=False, sleep_time=20,
+                                retry_count=2):
+        """Check if service was been started after a given time.
+
+        Args:
+          sentry_unit (sentry): The sentry unit to check for the service on
+          mtime (float): The epoch time to check against
+          service (string): service name to look for in process table
+          pgrep_full (boolean): Use full command line search mode with pgrep
+          sleep_time (int): Seconds to sleep before looking for process
+          retry_count (int): If service is not found, how many times to retry
+
+        Returns:
+          bool: True if service found and its start time it newer than mtime,
+                False if service is older than mtime or if service was
+                not found.
+        """
+        self.log.debug('Checking %s restarted since %s' % (service, mtime))
+        time.sleep(sleep_time)
+        proc_start_time = self._get_proc_start_time(sentry_unit, service,
+                                                    pgrep_full)
+        while retry_count > 0 and not proc_start_time:
+            self.log.debug('No pid file found for service %s, will retry %i '
+                           'more times' % (service, retry_count))
+            time.sleep(30)
+            proc_start_time = self._get_proc_start_time(sentry_unit, service,
+                                                        pgrep_full)
+            retry_count = retry_count - 1
+
+        if not proc_start_time:
+            self.log.warn('No proc start time found, assuming service did '
+                          'not start')
+            return False
+        if proc_start_time >= mtime:
+            self.log.debug('proc start time is newer than provided mtime'
+                           '(%s >= %s)' % (proc_start_time, mtime))
+            return True
+        else:
+            self.log.warn('proc start time (%s) is older than provided mtime '
+                          '(%s), service did not restart' % (proc_start_time,
+                                                             mtime))
+            return False
+
+    def config_updated_since(self, sentry_unit, filename, mtime,
+                             sleep_time=20):
+        """Check if file was modified after a given time.
+
+        Args:
+          sentry_unit (sentry): The sentry unit to check the file mtime on
+          filename (string): The file to check mtime of
+          mtime (float): The epoch time to check against
+          sleep_time (int): Seconds to sleep before looking for process
+
+        Returns:
+          bool: True if file was modified more recently than mtime, False if
+                file was modified before mtime,
+        """
+        self.log.debug('Checking %s updated since %s' % (filename, mtime))
+        time.sleep(sleep_time)
+        file_mtime = self._get_file_mtime(sentry_unit, filename)
+        if file_mtime >= mtime:
+            self.log.debug('File mtime is newer than provided mtime '
+                           '(%s >= %s)' % (file_mtime, mtime))
+            return True
+        else:
+            self.log.warn('File mtime %s is older than provided mtime %s'
+                          % (file_mtime, mtime))
+            return False
+
+    def validate_service_config_changed(self, sentry_unit, mtime, service,
+                                        filename, pgrep_full=False,
+                                        sleep_time=20, retry_count=2):
+        """Check service and file were updated after mtime
+
+        Args:
+          sentry_unit (sentry): The sentry unit to check for the service on
+          mtime (float): The epoch time to check against
+          service (string): service name to look for in process table
+          filename (string): The file to check mtime of
+          pgrep_full (boolean): Use full command line search mode with pgrep
+          sleep_time (int): Seconds to sleep before looking for process
+          retry_count (int): If service is not found, how many times to retry
+
+        Typical Usage:
+            u = OpenStackAmuletUtils(ERROR)
+            ...
+            mtime = u.get_sentry_time(self.cinder_sentry)
+            self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
+            if not u.validate_service_config_changed(self.cinder_sentry,
+                                                     mtime,
+                                                     'cinder-api',
+                                                     '/etc/cinder/cinder.conf')
+                amulet.raise_status(amulet.FAIL, msg='update failed')
+        Returns:
+          bool: True if both service and file where updated/restarted after
+                mtime, False if service is older than mtime or if service was
+                not found or if filename was modified before mtime.
+        """
+        self.log.debug('Checking %s restarted since %s' % (service, mtime))
+        time.sleep(sleep_time)
+        service_restart = self.service_restarted_since(sentry_unit, mtime,
+                                                       service,
+                                                       pgrep_full=pgrep_full,
+                                                       sleep_time=0,
+                                                       retry_count=retry_count)
+        config_update = self.config_updated_since(sentry_unit, filename, mtime,
+                                                  sleep_time=0)
+        return service_restart and config_update
+
+    def get_sentry_time(self, sentry_unit):
+        """Return current epoch time on a sentry"""
+        cmd = "date +'%s'"
+        return float(sentry_unit.run(cmd)[0])
+
+    def relation_error(self, name, data):
+        return 'unexpected relation data in {} - {}'.format(name, data)
+
+    def endpoint_error(self, name, data):
+        return 'unexpected endpoint data in {} - {}'.format(name, data)

tests/charmhelpers/contrib/openstack/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

tests/charmhelpers/contrib/openstack/amulet/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -0,0 +1,137 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import six
+from collections import OrderedDict
+from charmhelpers.contrib.amulet.deployment import (
+    AmuletDeployment
+)
+
+
+class OpenStackAmuletDeployment(AmuletDeployment):
+    """OpenStack amulet deployment.
+
+       This class inherits from AmuletDeployment and has additional support
+       that is specifically for use by OpenStack charms.
+       """
+
+    def __init__(self, series=None, openstack=None, source=None, stable=True):
+        """Initialize the deployment environment."""
+        super(OpenStackAmuletDeployment, self).__init__(series)
+        self.openstack = openstack
+        self.source = source
+        self.stable = stable
+        # Note(coreycb): this needs to be changed when new next branches come
+        # out.
+        self.current_next = "trusty"
+
+    def _determine_branch_locations(self, other_services):
+        """Determine the branch locations for the other services.
+
+           Determine if the local branch being tested is derived from its
+           stable or next (dev) branch, and based on this, use the corresonding
+           stable or next branches for the other_services."""
+        base_charms = ['mysql', 'mongodb']
+
+        if self.stable:
+            for svc in other_services:
+                temp = 'lp:charms/{}'
+                svc['location'] = temp.format(svc['name'])
+        else:
+            for svc in other_services:
+                if svc['name'] in base_charms:
+                    temp = 'lp:charms/{}'
+                    svc['location'] = temp.format(svc['name'])
+                else:
+                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
+                    svc['location'] = temp.format(self.current_next,
+                                                  svc['name'])
+        return other_services
+
+    def _add_services(self, this_service, other_services):
+        """Add services to the deployment and set openstack-origin/source."""
+        other_services = self._determine_branch_locations(other_services)
+
+        super(OpenStackAmuletDeployment, self)._add_services(this_service,
+                                                             other_services)
+
+        services = other_services
+        services.append(this_service)
+        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+                      'ceph-osd', 'ceph-radosgw']
+        # Openstack subordinate charms do not expose an origin option as that
+        # is controlled by the principle
+        ignore = ['neutron-openvswitch']
+
+        if self.openstack:
+            for svc in services:
+                if svc['name'] not in use_source + ignore:
+                    config = {'openstack-origin': self.openstack}
+                    self.d.configure(svc['name'], config)
+
+        if self.source:
+            for svc in services:
+                if svc['name'] in use_source and svc['name'] not in ignore:
+                    config = {'source': self.source}
+                    self.d.configure(svc['name'], config)
+
+    def _configure_services(self, configs):
+        """Configure all of the services."""
+        for service, config in six.iteritems(configs):
+            self.d.configure(service, config)
+
+    def _get_openstack_release(self):
+        """Get openstack release.
+
+           Return an integer representing the enum value of the openstack
+           release.
+           """
+        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+         self.precise_havana, self.precise_icehouse,
+         self.trusty_icehouse, self.trusty_juno, self.trusty_kilo,
+         self.utopic_juno, self.vivid_kilo) = range(10)
+        releases = {
+            ('precise', None): self.precise_essex,
+            ('precise', 'cloud:precise-folsom'): self.precise_folsom,
+            ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
+            ('precise', 'cloud:precise-havana'): self.precise_havana,
+            ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
+            ('trusty', None): self.trusty_icehouse,
+            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
+            ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
+            ('utopic', None): self.utopic_juno,
+            ('vivid', None): self.vivid_kilo}
+        return releases[(self.series, self.openstack)]
+
+    def _get_openstack_release_string(self):
+        """Get openstack release string.
+
+           Return a string representing the openstack release.
+           """
+        releases = OrderedDict([
+            ('precise', 'essex'),
+            ('quantal', 'folsom'),
+            ('raring', 'grizzly'),
+            ('saucy', 'havana'),
+            ('trusty', 'icehouse'),
+            ('utopic', 'juno'),
+            ('vivid', 'kilo'),
+        ])
+        if self.openstack:
+            os_origin = self.openstack.split(':')[1]
+            return os_origin.split('%s-' % self.series)[1].split('/')[0]
+        else:
+            return releases[self.series]

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -0,0 +1,294 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import time
+import urllib
+
+import glanceclient.v1.client as glance_client
+import keystoneclient.v2_0 as keystone_client
+import novaclient.v1_1.client as nova_client
+
+import six
+
+from charmhelpers.contrib.amulet.utils import (
+    AmuletUtils
+)
+
+DEBUG = logging.DEBUG
+ERROR = logging.ERROR
+
+
+class OpenStackAmuletUtils(AmuletUtils):
+    """OpenStack amulet utilities.
+
+       This class inherits from AmuletUtils and has additional support
+       that is specifically for use by OpenStack charms.
+       """
+
+    def __init__(self, log_level=ERROR):
+        """Initialize the deployment environment."""
+        super(OpenStackAmuletUtils, self).__init__(log_level)
+
+    def validate_endpoint_data(self, endpoints, admin_port, internal_port,
+                               public_port, expected):
+        """Validate endpoint data.
+
+           Validate actual endpoint data vs expected endpoint data. The ports
+           are used to find the matching endpoint.
+           """
+        found = False
+        for ep in endpoints:
+            self.log.debug('endpoint: {}'.format(repr(ep)))
+            if (admin_port in ep.adminurl and
+                    internal_port in ep.internalurl and
+                    public_port in ep.publicurl):
+                found = True
+                actual = {'id': ep.id,
+                          'region': ep.region,
+                          'adminurl': ep.adminurl,
+                          'internalurl': ep.internalurl,
+                          'publicurl': ep.publicurl,
+                          'service_id': ep.service_id}
+                ret = self._validate_dict_data(expected, actual)
+                if ret:
+                    return 'unexpected endpoint data - {}'.format(ret)
+
+        if not found:
+            return 'endpoint not found'
+
+    def validate_svc_catalog_endpoint_data(self, expected, actual):
+        """Validate service catalog endpoint data.
+
+           Validate a list of actual service catalog endpoints vs a list of
+           expected service catalog endpoints.
+           """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        for k, v in six.iteritems(expected):
+            if k in actual:
+                ret = self._validate_dict_data(expected[k][0], actual[k][0])
+                if ret:
+                    return self.endpoint_error(k, ret)
+            else:
+                return "endpoint {} does not exist".format(k)
+        return ret
+
+    def validate_tenant_data(self, expected, actual):
+        """Validate tenant data.
+
+           Validate a list of actual tenant data vs list of expected tenant
+           data.
+           """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        for e in expected:
+            found = False
+            for act in actual:
+                a = {'enabled': act.enabled, 'description': act.description,
+                     'name': act.name, 'id': act.id}
+                if e['name'] == a['name']:
+                    found = True
+                    ret = self._validate_dict_data(e, a)
+                    if ret:
+                        return "unexpected tenant data - {}".format(ret)
+            if not found:
+                return "tenant {} does not exist".format(e['name'])
+        return ret
+
+    def validate_role_data(self, expected, actual):
+        """Validate role data.
+
+           Validate a list of actual role data vs a list of expected role
+           data.
+           """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        for e in expected:
+            found = False
+            for act in actual:
+                a = {'name': act.name, 'id': act.id}
+                if e['name'] == a['name']:
+                    found = True
+                    ret = self._validate_dict_data(e, a)
+                    if ret:
+                        return "unexpected role data - {}".format(ret)
+            if not found:
+                return "role {} does not exist".format(e['name'])
+        return ret
+
+    def validate_user_data(self, expected, actual):
+        """Validate user data.
+
+           Validate a list of actual user data vs a list of expected user
+           data.
+           """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        for e in expected:
+            found = False
+            for act in actual:
+                a = {'enabled': act.enabled, 'name': act.name,
+                     'email': act.email, 'tenantId': act.tenantId,
+                     'id': act.id}
+                if e['name'] == a['name']:
+                    found = True
+                    ret = self._validate_dict_data(e, a)
+                    if ret:
+                        return "unexpected user data - {}".format(ret)
+            if not found:
+                return "user {} does not exist".format(e['name'])
+        return ret
+
+    def validate_flavor_data(self, expected, actual):
+        """Validate flavor data.
+
+           Validate a list of actual flavors vs a list of expected flavors.
+           """
+        self.log.debug('actual: {}'.format(repr(actual)))
+        act = [a.name for a in actual]
+        return self._validate_list_data(expected, act)
+
+    def tenant_exists(self, keystone, tenant):
+        """Return True if tenant exists."""
+        return tenant in [t.name for t in keystone.tenants.list()]
+
+    def authenticate_keystone_admin(self, keystone_sentry, user, password,
+                                    tenant):
+        """Authenticates admin user with the keystone admin endpoint."""
+        unit = keystone_sentry
+        service_ip = unit.relation('shared-db',
+                                   'mysql:shared-db')['private-address']
+        ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
+        return keystone_client.Client(username=user, password=password,
+                                      tenant_name=tenant, auth_url=ep)
+
+    def authenticate_keystone_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with the keystone public endpoint."""
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return keystone_client.Client(username=user, password=password,
+                                      tenant_name=tenant, auth_url=ep)
+
+    def authenticate_glance_admin(self, keystone):
+        """Authenticates admin user with glance."""
+        ep = keystone.service_catalog.url_for(service_type='image',
+                                              endpoint_type='adminURL')
+        return glance_client.Client(ep, token=keystone.auth_token)
+
+    def authenticate_nova_user(self, keystone, user, password, tenant):
+        """Authenticates a regular user with nova-api."""
+        ep = keystone.service_catalog.url_for(service_type='identity',
+                                              endpoint_type='publicURL')
+        return nova_client.Client(username=user, api_key=password,
+                                  project_id=tenant, auth_url=ep)
+
+    def create_cirros_image(self, glance, image_name):
+        """Download the latest cirros image and upload it to glance."""
+        http_proxy = os.getenv('AMULET_HTTP_PROXY')
+        self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy))
+        if http_proxy:
+            proxies = {'http': http_proxy}
+            opener = urllib.FancyURLopener(proxies)
+        else:
+            opener = urllib.FancyURLopener()
+
+        f = opener.open("http://download.cirros-cloud.net/version/released")
+        version = f.read().strip()
+        cirros_img = "cirros-{}-x86_64-disk.img".format(version)
+        local_path = os.path.join('tests', cirros_img)
+
+        if not os.path.exists(local_path):
+            cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
+                                                  version, cirros_img)
+            opener.retrieve(cirros_url, local_path)
+        f.close()
+
+        with open(local_path) as f:
+            image = glance.images.create(name=image_name, is_public=True,
+                                         disk_format='qcow2',
+                                         container_format='bare', data=f)
+        count = 1
+        status = image.status
+        while status != 'active' and count < 10:
+            time.sleep(3)
+            image = glance.images.get(image.id)
+            status = image.status
+            self.log.debug('image status: {}'.format(status))
+            count += 1
+
+        if status != 'active':
+            self.log.error('image creation timed out')
+            return None
+
+        return image
+
+    def delete_image(self, glance, image):
+        """Delete the specified image."""
+        num_before = len(list(glance.images.list()))
+        glance.images.delete(image)
+
+        count = 1
+        num_after = len(list(glance.images.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(glance.images.list()))
+            self.log.debug('number of images: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('image deletion timed out')
+            return False
+
+        return True
+
+    def create_instance(self, nova, image_name, instance_name, flavor):
+        """Create the specified instance."""
+        image = nova.images.find(name=image_name)
+        flavor = nova.flavors.find(name=flavor)
+        instance = nova.servers.create(name=instance_name, image=image,
+                                       flavor=flavor)
+
+        count = 1
+        status = instance.status
+        while status != 'ACTIVE' and count < 60:
+            time.sleep(3)
+            instance = nova.servers.get(instance.id)
+            status = instance.status
+            self.log.debug('instance status: {}'.format(status))
+            count += 1
+
+        if status != 'ACTIVE':
+            self.log.error('instance creation timed out')
+            return None
+
+        return instance
+
+    def delete_instance(self, nova, instance):
+        """Delete the specified instance."""
+        num_before = len(list(nova.servers.list()))
+        nova.servers.delete(instance)
+
+        count = 1
+        num_after = len(list(nova.servers.list()))
+        while num_after != (num_before - 1) and count < 10:
+            time.sleep(3)
+            num_after = len(list(nova.servers.list()))
+            self.log.debug('number of instances: {}'.format(num_after))
+            count += 1
+
+        if num_after != (num_before - 1):
+            self.log.error('instance deletion timed out')
+            return False
+
+        return True

unit_tests/test_percona_hooks.py

@@ -0,0 +1,65 @@
+import mock
+import sys
+from test_utils import CharmTestCase
+
+sys.modules['MySQLdb'] = mock.Mock()
+import percona_hooks as hooks
+
+TO_PATCH = ['log', 'config',
+            'get_db_helper',
+            'relation_ids',
+            'relation_set']
+
+
+class TestHaRelation(CharmTestCase):
+    def setUp(self):
+        CharmTestCase.setUp(self, hooks, TO_PATCH)
+
+    @mock.patch('sys.exit')
+    def test_relation_not_configured(self, exit_):
+        self.config.return_value = None
+
+        class MyError(Exception):
+            pass
+
+        def f(x):
+            raise MyError(x)
+        exit_.side_effect = f
+        self.assertRaises(MyError, hooks.ha_relation_joined)
+
+    def test_resources(self):
+        self.relation_ids.return_value = ['ha:1']
+        password = 'ubuntu'
+        helper = mock.Mock()
+        attrs = {'get_mysql_password.return_value': password}
+        helper.configure_mock(**attrs)
+        self.get_db_helper.return_value = helper
+        self.test_config.set('vip', '10.0.3.3')
+        self.test_config.set('sst-password', password)
+        def f(k):
+            return self.test_config.get(k)
+
+        self.config.side_effect = f
+        hooks.ha_relation_joined()
+
+        resources = {'res_mysql_vip': 'ocf:heartbeat:IPaddr2',
+                     'res_mysql_monitor': 'ocf:percona:mysql_monitor'}
+        resource_params = {'res_mysql_vip': ('params ip="10.0.3.3" '
+                                             'cidr_netmask="24" '
+                                             'nic="eth0"'),
+                           'res_mysql_monitor':
+                               hooks.RES_MONITOR_PARAMS % {'sstpass': 'ubuntu'}}
+        groups = {'grp_percona_cluster': 'res_mysql_vip'}
+
+        clones = {'cl_mysql_monitor': 'res_mysql_monitor meta interleave=true'}
+
+        colocations = {'vip_mysqld': 'inf: grp_percona_cluster cl_mysql_monitor'}
+
+        locations = {'loc_percona_cluster':
+                     'grp_percona_cluster rule inf: writable eq 1'}
+
+        self.relation_set.assert_called_with(
+            relation_id='ha:1', corosync_bindiface=f('ha-bindiface'),
+            corosync_mcastport=f('ha-mcastport'), resources=resources,
+            resource_params=resource_params, groups=groups,
+            clones=clones, colocations=colocations, locations=locations)

unit_tests/test_utils.py

@@ -0,0 +1,121 @@
+import logging
+import unittest
+import os
+import yaml
+
+from contextlib import contextmanager
+from mock import patch, MagicMock
+
+
+def load_config():
+    '''
+    Walk backwords from __file__ looking for config.yaml, load and return the
+    'options' section'
+    '''
+    config = None
+    f = __file__
+    while config is None:
+        d = os.path.dirname(f)
+        if os.path.isfile(os.path.join(d, 'config.yaml')):
+            config = os.path.join(d, 'config.yaml')
+            break
+        f = d
+
+    if not config:
+        logging.error('Could not find config.yaml in any parent directory '
+                      'of %s. ' % file)
+        raise Exception
+
+    return yaml.safe_load(open(config).read())['options']
+
+
+def get_default_config():
+    '''
+    Load default charm config from config.yaml return as a dict.
+    If no default is set in config.yaml, its value is None.
+    '''
+    default_config = {}
+    config = load_config()
+    for k, v in config.iteritems():
+        if 'default' in v:
+            default_config[k] = v['default']
+        else:
+            default_config[k] = None
+    return default_config
+
+
+class CharmTestCase(unittest.TestCase):
+
+    def setUp(self, obj, patches):
+        super(CharmTestCase, self).setUp()
+        self.patches = patches
+        self.obj = obj
+        self.test_config = TestConfig()
+        self.test_relation = TestRelation()
+        self.patch_all()
+
+    def patch(self, method):
+        _m = patch.object(self.obj, method)
+        mock = _m.start()
+        self.addCleanup(_m.stop)
+        return mock
+
+    def patch_all(self):
+        for method in self.patches:
+            setattr(self, method, self.patch(method))
+
+
+class TestConfig(object):
+
+    def __init__(self):
+        self.config = get_default_config()
+
+    def get(self, attr=None):
+        if not attr:
+            return self.get_all()
+        try:
+            return self.config[attr]
+        except KeyError:
+            return None
+
+    def get_all(self):
+        return self.config
+
+    def set(self, attr, value):
+        if attr not in self.config:
+            raise KeyError
+        self.config[attr] = value
+
+
+class TestRelation(object):
+
+    def __init__(self, relation_data={}):
+        self.relation_data = relation_data
+
+    def set(self, relation_data):
+        self.relation_data = relation_data
+
+    def get(self, attr=None, unit=None, rid=None):
+        if attr is None:
+            return self.relation_data
+        elif attr in self.relation_data:
+            return self.relation_data[attr]
+        return None
+
+
+@contextmanager
+def patch_open():
+    '''Patch open() to allow mocking both open() itself and the file that is
+    yielded.
+
+    Yields the mock for "open" and "file", respectively.'''
+    mock_open = MagicMock(spec=open)
+    mock_file = MagicMock(spec=file)
+
+    @contextmanager
+    def stub_open(*args, **kwargs):
+        mock_open(*args, **kwargs)
+        yield mock_file
+
+    with patch('__builtin__.open', stub_open):
+        yield mock_open, mock_file