diff --git a/.bzrignore b/.bzrignore index 4ff08cd..0c2be0d 100644 --- a/.bzrignore +++ b/.bzrignore @@ -2,3 +2,6 @@ bin .coverage .pydevproject .project +*.pyc +*.pyo +__pycache__ diff --git a/Makefile b/Makefile index 858c03f..d578cb5 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,13 @@ lint: unit_test: @$(PYTHON) /usr/bin/nosetests --nologcapture unit_tests +test: + @echo Starting amulet tests... + #NOTE(beisner): can remove -v after bug 1320357 is fixed + # https://bugs.launchpad.net/amulet/+bug/1320357 + # @juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700 + echo "Tests disables; http://pad.lv/1446169" + bin/charm_helpers_sync.py: @mkdir -p bin @bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py \ @@ -16,6 +23,7 @@ bin/charm_helpers_sync.py: sync: bin/charm_helpers_sync.py @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml + @$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers-tests.yaml publish: lint bzr push lp:charms/trusty/percona-cluster diff --git a/charm-helpers-tests.yaml b/charm-helpers-tests.yaml new file mode 100644 index 0000000..48b12f6 --- /dev/null +++ b/charm-helpers-tests.yaml @@ -0,0 +1,5 @@ +branch: lp:charm-helpers +destination: tests/charmhelpers +include: + - contrib.amulet + - contrib.openstack.amulet diff --git a/copyright b/copyright index 1632584..4c35cfe 100644 --- a/copyright +++ b/copyright @@ -2,16 +2,28 @@ Format: http://dep.debian.net/deps/dep5/ Files: * Copyright: Copyright 2011, Canonical Ltd., All Rights Reserved. -License: GPL-3 - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - . - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - . +License: GPL-2 + +Files: ocf/percona/mysql_monitor +Copyright: Copyright (c) 2013, Percona inc., Yves Trudeau, Michael Coburn +License: GPL-2 + +License: GPL-2 + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it would be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + Further, this software is distributed without any warranty that it is + free of the rightful claim of any third person regarding infringement + or the like. Any license provided herein, whether implied or + otherwise, applies only to this software file. Patent licenses, if + any, provided herein do not apply to combinations of this program with + other software, or any other product whatsoever. + You should have received a copy of the GNU General Public License - along with this program. If not, see . + along with this program; if not, write the Free Software Foundation, + Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. diff --git a/hooks/percona_hooks.py b/hooks/percona_hooks.py index 7c4b14e..7bdd39d 100755 --- a/hooks/percona_hooks.py +++ b/hooks/percona_hooks.py @@ -50,6 +50,7 @@ from percona_utils import ( assert_charm_supports_ipv6, unit_sorted, get_db_helper, + install_mysql_ocf, ) from charmhelpers.contrib.database.mysql import ( PerconaClusterHelper, @@ -72,6 +73,13 @@ from charmhelpers.contrib.network.ip import ( hooks = Hooks() LEADER_RES = 'grp_percona_cluster' +RES_MONITOR_PARAMS = ('params user="sstuser" password="%(sstpass)s" ' + 'pid="/var/run/mysqld/mysqld.pid" ' + 'socket="/var/run/mysqld/mysqld.sock" ' + 'max_slave_lag="5" ' + 'cluster_type="pxc" ' + 'op monitor interval="1s" timeout="30s" ' + 'OCF_CHECK_LEVEL="1"') @hooks.hook('install') @@ -155,6 +163,13 @@ def config_changed(): for unit in related_units(r_id): shared_db_changed(r_id, unit) + # (re)install pcmkr agent + install_mysql_ocf() + + if relation_ids('ha'): + # make sure all the HA resources are (re)created + ha_relation_joined() + @hooks.hook('cluster-relation-joined') def cluster_joined(relation_id=None): @@ -387,17 +402,34 @@ def ha_relation_joined(): vip_params = 'params ip="%s" cidr_netmask="%s" nic="%s"' % \ (vip, vip_cidr, vip_iface) - resources = {'res_mysql_vip': res_mysql_vip} - resource_params = {'res_mysql_vip': vip_params} + resources = {'res_mysql_vip': res_mysql_vip, + 'res_mysql_monitor': 'ocf:percona:mysql_monitor'} + db_helper = get_db_helper() + cfg_passwd = config('sst-password') + sstpsswd = db_helper.get_mysql_password(username='sstuser', + password=cfg_passwd) + resource_params = {'res_mysql_vip': vip_params, + 'res_mysql_monitor': + RES_MONITOR_PARAMS % {'sstpass': sstpsswd}} groups = {'grp_percona_cluster': 'res_mysql_vip'} + clones = {'cl_mysql_monitor': 'res_mysql_monitor meta interleave=true'} + + colocations = {'vip_mysqld': 'inf: grp_percona_cluster cl_mysql_monitor'} + + locations = {'loc_percona_cluster': + 'grp_percona_cluster rule inf: writable eq 1'} + for rel_id in relation_ids('ha'): relation_set(relation_id=rel_id, corosync_bindiface=corosync_bindiface, corosync_mcastport=corosync_mcastport, resources=resources, resource_params=resource_params, - groups=groups) + groups=groups, + clones=clones, + colocations=colocations, + locations=locations) @hooks.hook('ha-relation-changed') diff --git a/hooks/percona_utils.py b/hooks/percona_utils.py index f7c92bf..8bf9fc7 100644 --- a/hooks/percona_utils.py +++ b/hooks/percona_utils.py @@ -4,10 +4,12 @@ from subprocess import Popen, PIPE import socket import tempfile import os +import shutil from charmhelpers.core.host import ( lsb_release ) from charmhelpers.core.hookenv import ( + charm_dir, unit_get, relation_ids, related_units, @@ -229,3 +231,18 @@ def unit_sorted(units): """Return a sorted list of unit names.""" return sorted( units, lambda a, b: cmp(int(a.split('/')[-1]), int(b.split('/')[-1]))) + + +def install_mysql_ocf(): + dest_dir = '/usr/lib/ocf/resource.d/percona/' + for fname in ['ocf/percona/mysql_monitor']: + src_file = os.path.join(charm_dir(), fname) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir) + + dest_file = os.path.join(dest_dir, os.path.basename(src_file)) + if not os.path.exists(dest_file): + log('Installing %s' % dest_file, level='INFO') + shutil.copy(src_file, dest_file) + else: + log("'%s' already exists, skipping" % dest_file, level='INFO') diff --git a/ocf/percona/mysql_monitor b/ocf/percona/mysql_monitor new file mode 100755 index 0000000..e60499b --- /dev/null +++ b/ocf/percona/mysql_monitor @@ -0,0 +1,636 @@ +#!/bin/bash +# +# +# MySQL_Monitor agent, set writeable and readable attributes based on the +# state of the local MySQL, running and read_only or not. The agent basis is +# the original "Dummy" agent written by Lars Marowsky-Brée and part of the +# Pacemaker distribution. Many functions are from mysql_prm. +# +# +# Copyright (c) 2013, Percona inc., Yves Trudeau, Michael Coburn +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of version 2 of the GNU General Public License as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it would be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# +# Further, this software is distributed without any warranty that it is +# free of the rightful claim of any third person regarding infringement +# or the like. Any license provided herein, whether implied or +# otherwise, applies only to this software file. Patent licenses, if +# any, provided herein do not apply to combinations of this program with +# other software, or any other product whatsoever. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write the Free Software Foundation, +# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA. +# +# Version: 20131119163921 +# +# See usage() function below for more details... +# +# OCF instance parameters: +# +# OCF_RESKEY_state +# OCF_RESKEY_user +# OCF_RESKEY_password +# OCF_RESKEY_client_binary +# OCF_RESKEY_pid +# OCF_RESKEY_socket +# OCF_RESKEY_reader_attribute +# OCF_RESKEY_reader_failcount +# OCF_RESKEY_writer_attribute +# OCF_RESKEY_max_slave_lag +# OCF_RESKEY_cluster_type +# +####################################################################### +# Initialization: + +: ${OCF_FUNCTIONS_DIR=${OCF_ROOT}/lib/heartbeat} +. ${OCF_FUNCTIONS_DIR}/ocf-shellfuncs + +####################################################################### + +HOSTOS=`uname` +if [ "X${HOSTOS}" = "XOpenBSD" ];then +OCF_RESKEY_client_binary_default="/usr/local/bin/mysql" +OCF_RESKEY_pid_default="/var/mysql/mysqld.pid" +OCF_RESKEY_socket_default="/var/run/mysql/mysql.sock" +else +OCF_RESKEY_client_binary_default="/usr/bin/mysql" +OCF_RESKEY_pid_default="/var/run/mysql/mysqld.pid" +OCF_RESKEY_socket_default="/var/lib/mysql/mysql.sock" +fi +OCF_RESKEY_reader_attribute_default="readable" +OCF_RESKEY_writer_attribute_default="writable" +OCF_RESKEY_reader_failcount_default="1" +OCF_RESKEY_user_default="root" +OCF_RESKEY_password_default="" +OCF_RESKEY_max_slave_lag_default="3600" +OCF_RESKEY_cluster_type_default="replication" + +: ${OCF_RESKEY_state=${HA_RSCTMP}/mysql-monitor-${OCF_RESOURCE_INSTANCE}.state} +: ${OCF_RESKEY_client_binary=${OCF_RESKEY_client_binary_default}} +: ${OCF_RESKEY_pid=${OCF_RESKEY_pid_default}} +: ${OCF_RESKEY_socket=${OCF_RESKEY_socket_default}} +: ${OCF_RESKEY_reader_attribute=${OCF_RESKEY_reader_attribute_default}} +: ${OCF_RESKEY_reader_failcount=${OCF_RESKEY_reader_failcount_default}} +: ${OCF_RESKEY_writer_attribute=${OCF_RESKEY_writer_attribute_default}} +: ${OCF_RESKEY_user=${OCF_RESKEY_user_default}} +: ${OCF_RESKEY_password=${OCF_RESKEY_password_default}} +: ${OCF_RESKEY_max_slave_lag=${OCF_RESKEY_max_slave_lag_default}} +: ${OCF_RESKEY_cluster_type=${OCF_RESKEY_cluster_type_default}} + +MYSQL="$OCF_RESKEY_client_binary -A -S $OCF_RESKEY_socket --connect_timeout=10 --user=$OCF_RESKEY_user --password=$OCF_RESKEY_password " +HOSTNAME=`uname -n` +CRM_ATTR="${HA_SBIN_DIR}/crm_attribute -N $HOSTNAME " + +meta_data() { + cat < + + +1.0 + + +This agent monitors the local MySQL instance and set the writable and readable +attributes according to what it finds. It checks if MySQL is running and if +it is read-only or not. + +Agent monitoring mysql + + + + +Location to store the resource state in. + +State file + + + + + +MySQL user to connect to the local MySQL instance to check the slave status and +if the read_only variable is set. It requires the replication client priviledge. + +MySQL user + + + + + +Password of the mysql user to connect to the local MySQL instance + +MySQL password + + + + + +MySQL Client Binary path. + +MySQL client binary path + + + + + +Unix socket to use in order to connect to MySQL on the host + +MySQL socket + + + + + +MySQL pid file, used to verify MySQL is running. + +MySQL pid file + + + + + +The reader attribute in the cib that can be used by location rules to allow or not +reader VIPs on a host. + +Reader attribute + + + + + +The reader attribute in the cib that can be used by location rules to allow or not +reader VIPs on a host. + +Writer attribute + + + + + +The maximum number of seconds a replication slave is allowed to lag +behind its master in order to have a reader VIP on it. + +Maximum time (seconds) a MySQL slave is allowed +to lag behind a master + + + + + +Type of cluster, three possible values: pxc, replication, read-only. "pxc" is +for Percona XtraDB cluster, it uses the clustercheck script and set the +reader_attribute and writer_attribute according to the return code. +"replication" checks the read-only state and the slave status, only writable +node(s) will get the writer_attribute (and the reader_attribute) and on the +read-only nodes, replication status will be checked and the reader_attribute set +according to the state. "read-only" will just check if the read-only variable, +if read/write, it will get both the writer_attribute and reader_attribute set, if +read-only it will get only the reader_attribute. + +Type of cluster + + + + + + + + + + + + + + + + +END +} + +####################################################################### +# Non API functions + +# Extract fields from slave status +parse_slave_info() { + # Extracts field $1 from result of "SHOW SLAVE STATUS\G" from file $2 + sed -ne "s/^.* $1: \(.*\)$/\1/p" < $2 +} + +# Read the slave status and +get_slave_info() { + + local mysql_options tmpfile + + if [ "$master_log_file" -a "$master_host" ]; then + # variables are already defined, get_slave_info has been run before + return $OCF_SUCCESS + else + tmpfile=`mktemp ${HA_RSCTMP}/check_slave.${OCF_RESOURCE_INSTANCE}.XXXXXX` + + mysql_run -Q -sw -O $MYSQL $MYSQL_OPTIONS_REPL \ + -e 'SHOW SLAVE STATUS\G' > $tmpfile + + if [ -s $tmpfile ]; then + master_host=`parse_slave_info Master_Host $tmpfile` + slave_sql=`parse_slave_info Slave_SQL_Running $tmpfile` + slave_io=`parse_slave_info Slave_IO_Running $tmpfile` + slave_io_state=`parse_slave_info Slave_IO_State $tmpfile` + last_errno=`parse_slave_info Last_Errno $tmpfile` + secs_behind=`parse_slave_info Seconds_Behind_Master $tmpfile` + ocf_log debug "MySQL instance has a non empty slave status" + else + # Instance produced an empty "SHOW SLAVE STATUS" output -- + # instance is not a slave + + ocf_log err "check_slave invoked on an instance that is not a replication slave." + rm -f $tmpfile + return $OCF_ERR_GENERIC + fi + rm -f $tmpfile + return $OCF_SUCCESS + fi +} + +get_read_only() { + # Check if read-only is set + local read_only_state + + read_only_state=`mysql_run -Q -sw -O $MYSQL -N $MYSQL_OPTIONS_REPL \ + -e "SHOW VARIABLES like 'read_only'" | awk '{print $2}'` + + if [ "$read_only_state" = "ON" ]; then + return 0 + else + return 1 + fi +} + +# get the attribute controlling the readers VIP +get_reader_attr() { + local attr_value + local rc + + attr_value=`$CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} --query -q` + rc=$? + if [ "$rc" -eq "0" ]; then + echo $attr_value + else + echo -1 + fi + +} + +# Set the attribute controlling the readers VIP +set_reader_attr() { + local curr_attr_value + + curr_attr_value=$(get_reader_attr) + + if [ "$1" -eq "0" ]; then + if [ "$curr_attr_value" -gt "0" ]; then + curr_attr_value=$((${curr_attr_value}-1)) + $CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} -v $curr_attr_value + else + $CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} -v 0 + fi + else + if [ "$curr_attr_value" -ne "$OCF_RESKEY_reader_failcount" ]; then + $CRM_ATTR -l reboot --name ${OCF_RESKEY_reader_attribute} -v $OCF_RESKEY_reader_failcount + fi + fi + +} + +# get the attribute controlling the writer VIP +get_writer_attr() { + local attr_value + local rc + + attr_value=`$CRM_ATTR -l reboot --name ${OCF_RESKEY_writer_attribute} --query -q` + rc=$? + if [ "$rc" -eq "0" ]; then + echo $attr_value + else + echo -1 + fi + +} + +# Set the attribute controlling the writer VIP +set_writer_attr() { + local curr_attr_value + + curr_attr_value=$(get_writer_attr) + + if [ "$1" -ne "$curr_attr_value" ]; then + if [ "$1" -eq "0" ]; then + $CRM_ATTR -l reboot --name ${OCF_RESKEY_writer_attribute} -v 0 + else + $CRM_ATTR -l reboot --name ${OCF_RESKEY_writer_attribute} -v 1 + fi + fi +} + +# +# mysql_run: Run a mysql command, log its output and return the proper error code. +# Usage: mysql_run [-Q] [-info|-warn|-err] [-O] [-sw] +# -Q: don't log the output of the command if it succeeds +# -info|-warn|-err: log the output of the command at given +# severity if it fails (defaults to err) +# -O: echo the output of the command +# -sw: Suppress 5.6 client warning when password is used on the command line +# Adapted from ocf_run. +# +mysql_run() { + local rc + local output outputfile + local verbose=1 + local returnoutput + local loglevel=err + local suppress_56_password_warning + local var + + for var in 1 2 3 4 + do + case "$1" in + "-Q") + verbose="" + shift 1;; + "-info"|"-warn"|"-err") + loglevel=`echo $1 | sed -e s/-//g` + shift 1;; + "-O") + returnoutput=1 + shift 1;; + "-sw") + suppress_56_password_warning=1 + shift 1;; + + *) + ;; + esac + done + + outputfile=`mktemp ${HA_RSCTMP}/mysql_run.${OCF_RESOURCE_INSTANCE}.XXXXXX` + error=`"$@" 2>&1 1>$outputfile` + rc=$? + if [ "$suppress_56_password_warning" -eq 1 ]; then + error=`echo "$error" | egrep -v '^Warning: Using a password on the command line'` + fi + output=`cat $outputfile` + rm -f $outputfile + + if [ $rc -eq 0 ]; then + if [ "$verbose" -a ! -z "$output" ]; then + ocf_log info "$output" + fi + + if [ "$returnoutput" -a ! -z "$output" ]; then + echo "$output" + fi + + MYSQL_LAST_ERR=$OCF_SUCCESS + return $OCF_SUCCESS + else + if [ ! -z "$error" ]; then + ocf_log $loglevel "$error" + regex='^ERROR ([[:digit:]]{4}).*' + if [[ $error =~ $regex ]]; then + mysql_code=${BASH_REMATCH[1]} + if [ -n "$mysql_code" ]; then + MYSQL_LAST_ERR=$mysql_code + return $rc + fi + fi + else + ocf_log $loglevel "command failed: $*" + fi + # No output to parse so return the standard exit code. + MYSQL_LAST_ERR=$rc + return $rc + fi +} + + + + +####################################################################### +# API functions + +mysql_monitor_usage() { + cat </dev/null 2>&1 + fi + + if [ $? -eq 0 ]; then + + case ${OCF_RESKEY_cluster_type} in + 'replication'|'REPLICATION') + if get_read_only; then + # a slave? + + set_writer_attr 0 + + get_slave_info + rc=$? + + if [ $rc -eq 0 ]; then + # show slave status is not empty + # Is there a master_log_file defined? (master_log_file is deleted + # by reset slave + if [ "$master_log_file" ]; then + # is read_only but no slave config... + + set_reader_attr 0 + + else + # has a slave config + + if [ "$slave_sql" = 'Yes' -a "$slave_io" = 'Yes' ]; then + # $secs_behind can be NULL so must be tested only + # if replication is OK + if [ $secs_behind -gt $OCF_RESKEY_max_slave_lag ]; then + set_reader_attr 0 + else + set_reader_attr 1 + fi + else + set_reader_attr 0 + fi + fi + else + # "SHOW SLAVE STATUS" returns an empty set if instance is not a + # replication slave + + set_reader_attr 0 + + fi + else + # host is RW + set_reader_attr 1 + set_writer_attr 1 + fi + ;; + + 'pxc'|'PXC') + pxcstat=`/usr/bin/clustercheck $OCF_RESKEY_user $OCF_RESKEY_password ` + if [ $? -eq 0 ]; then + set_reader_attr 1 + set_writer_attr 1 + else + set_reader_attr 0 + set_writer_attr 0 + fi + + ;; + + 'read-only'|'READ-ONLY') + if get_read_only; then + set_reader_attr 1 + set_writer_attr 0 + else + set_reader_attr 1 + set_writer_attr 1 + fi + ;; + + esac + else + ocf_log $1 "MySQL is not running, but there is a pidfile" + set_reader_attr 0 + set_writer_attr 0 + fi + else + ocf_log $1 "MySQL is not running" + set_reader_attr 0 + set_writer_attr 0 + fi +} + +mysql_monitor_monitor() { + # Monitor _MUST!_ differentiate correctly between running + # (SUCCESS), failed (ERROR) or _cleanly_ stopped (NOT RUNNING). + # That is THREE states, not just yes/no. + + if [ -f ${OCF_RESKEY_state} ]; then + return $OCF_SUCCESS + fi + if false ; then + return $OCF_ERR_GENERIC + fi + return $OCF_NOT_RUNNING +} + +mysql_monitor_validate() { + + # Is the state directory writable? + state_dir=`dirname "$OCF_RESKEY_state"` + touch "$state_dir/$$" + if [ $? != 0 ]; then + return $OCF_ERR_ARGS + fi + rm "$state_dir/$$" + + return $OCF_SUCCESS +} + +########################################################################## +# If DEBUG_LOG is set, make this resource agent easy to debug: set up the +# debug log and direct all output to it. Otherwise, redirect to /dev/null. +# The log directory must be a directory owned by root, with permissions 0700, +# and the log must be writable and not a symlink. +########################################################################## +DEBUG_LOG="/tmp/mysql_monitor.ocf.ra.debug/log" +if [ "${DEBUG_LOG}" -a -w "${DEBUG_LOG}" -a ! -L "${DEBUG_LOG}" ]; then + DEBUG_LOG_DIR="${DEBUG_LOG%/*}" + if [ -d "${DEBUG_LOG_DIR}" ]; then + exec 9>>"$DEBUG_LOG" + exec 2>&9 + date >&9 + echo "$*" >&9 + env | grep OCF_ | sort >&9 + set -x + else + exec 9>/dev/null + fi +fi + + +case $__OCF_ACTION in +meta-data) meta_data + exit $OCF_SUCCESS + ;; +start) mysql_monitor_start;; +stop) mysql_monitor_stop;; +monitor) mysql_monitor + mysql_monitor_monitor;; +migrate_to) ocf_log info "Migrating ${OCF_RESOURCE_INSTANCE} to ${OCF_RESKEY_CRM_meta_migrate_target}." + mysql_monitor_stop + ;; +migrate_from) ocf_log info "Migrating ${OCF_RESOURCE_INSTANCE} from ${OCF_RESKEY_CRM_meta_migrate_source}." + mysql_monitor_start + ;; +reload) ocf_log info "Reloading ${OCF_RESOURCE_INSTANCE} ..." + ;; +validate-all) mysql_monitor_validate;; +usage|help) mysql_monitor_usage + exit $OCF_SUCCESS + ;; +*) mysql_monitor_usage + exit $OCF_ERR_UNIMPLEMENTED + ;; +esac +rc=$? +ocf_log debug "${OCF_RESOURCE_INSTANCE} $__OCF_ACTION : $rc" +exit $rc + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3f7bd91 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[nosetests] +verbosity=2 +with-coverage=1 +cover-erase=1 +cover-package=hooks + diff --git a/templates/my.cnf b/templates/my.cnf index 6df56b5..2acee48 100644 --- a/templates/my.cnf +++ b/templates/my.cnf @@ -11,6 +11,7 @@ wsrep_provider_options = {{ wsrep_provider_options }} datadir=/var/lib/mysql user=mysql +pid_file = /var/run/mysqld/mysqld.pid # Path to Galera library wsrep_provider=/usr/lib/libgalera_smm.so diff --git a/tests/00-setup.sh b/tests/00-setup.sh new file mode 100755 index 0000000..ed1c292 --- /dev/null +++ b/tests/00-setup.sh @@ -0,0 +1,29 @@ +#!/bin/bash -x +# The script installs amulet and other tools needed for the amulet tests. + +# Get the status of the amulet package, this returns 0 of package is installed. +dpkg -s amulet +if [ $? -ne 0 ]; then + # Install the Amulet testing harness. + sudo add-apt-repository -y ppa:juju/stable + sudo apt-get update + sudo apt-get install -y -q amulet juju-core charm-tools +fi + + +PACKAGES="python3 python3-yaml" +for pkg in $PACKAGES; do + dpkg -s python3 + if [ $? -ne 0 ]; then + sudo apt-get install -y -q $pkg + fi +done + + +#if [ ! -f "$(dirname $0)/../local.yaml" ]; then +# echo "To run these amulet tests a vip is needed, create a file called \ +#local.yaml in the charm dir, this file must contain a 'vip', if you're \ +#using the local provider with lxc you could use a free IP from the range \ +#10.0.3.0/24" +# exit 1 +#fi diff --git a/tests/10-deploy_test.py b/tests/10-deploy_test.py new file mode 100755 index 0000000..312e9c1 --- /dev/null +++ b/tests/10-deploy_test.py @@ -0,0 +1,29 @@ +#!/usr/bin/python3 +# test percona-cluster (3 nodes) + +import basic_deployment +import time + + +class ThreeNode(basic_deployment.BasicDeployment): + def __init__(self): + super(ThreeNode, self).__init__(units=3) + + def run(self): + super(ThreeNode, self).run() + # we are going to kill the master + old_master = self.master_unit + self.master_unit.run('sudo poweroff') + + time.sleep(10) # give some time to pacemaker to react + new_master = self.find_master() + assert new_master is not None, "master unit not found" + assert (new_master.info['public-address'] != + old_master.info['public-address']) + + assert self.is_port_open(address=self.vip), 'cannot connect to vip' + + +if __name__ == "__main__": + t = ThreeNode() + t.run() diff --git a/tests/20-broken-mysqld.py b/tests/20-broken-mysqld.py new file mode 100755 index 0000000..520126f --- /dev/null +++ b/tests/20-broken-mysqld.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 +# test percona-cluster (3 nodes) + +import basic_deployment +import time + + +class ThreeNode(basic_deployment.BasicDeployment): + def __init__(self): + super(ThreeNode, self).__init__(units=3) + + def run(self): + super(ThreeNode, self).run() + # we are going to kill the master + old_master = self.master_unit + print('stopping mysql in %s' % str(self.master_unit.info)) + self.master_unit.run('sudo service mysql stop') + + print('looking for the new master') + i = 0 + changed = False + while i < 10 and not changed: + i += 1 + time.sleep(5) # give some time to pacemaker to react + new_master = self.find_master() + + if (new_master and new_master.info['unit_name'] != + old_master.info['unit_name']): + changed = True + + assert changed, "The master didn't change" + + assert self.is_port_open(address=self.vip), 'cannot connect to vip' + + +if __name__ == "__main__": + t = ThreeNode() + t.run() diff --git a/tests/30-kill-9-mysqld.py b/tests/30-kill-9-mysqld.py new file mode 100755 index 0000000..7ba58e9 --- /dev/null +++ b/tests/30-kill-9-mysqld.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 +# test percona-cluster (3 nodes) + +import basic_deployment +import time + + +class ThreeNode(basic_deployment.BasicDeployment): + def __init__(self): + super(ThreeNode, self).__init__(units=3) + + def run(self): + super(ThreeNode, self).run() + # we are going to kill the master + old_master = self.master_unit + print('kill-9 mysqld in %s' % str(self.master_unit.info)) + self.master_unit.run('sudo killall -9 mysqld') + + print('looking for the new master') + i = 0 + changed = False + while i < 10 and not changed: + i += 1 + time.sleep(5) # give some time to pacemaker to react + new_master = self.find_master() + + if (new_master and new_master.info['unit_name'] != + old_master.info['unit_name']): + changed = True + + assert changed, "The master didn't change" + + assert self.is_port_open(address=self.vip), 'cannot connect to vip' + + +if __name__ == "__main__": + t = ThreeNode() + t.run() diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py new file mode 100644 index 0000000..b97a892 --- /dev/null +++ b/tests/basic_deployment.py @@ -0,0 +1,151 @@ +import amulet +import os +import time +import telnetlib +import unittest +import yaml +from charmhelpers.contrib.openstack.amulet.deployment import ( + OpenStackAmuletDeployment +) + + +class BasicDeployment(OpenStackAmuletDeployment): + def __init__(self, vip=None, units=1, series="trusty", openstack=None, + source=None, stable=False): + super(BasicDeployment, self).__init__(series, openstack, source, + stable) + self.units = units + self.master_unit = None + self.vip = None + if vip: + self.vip = vip + elif 'AMULET_OS_VIP' in os.environ: + self.vip = os.environ.get('AMULET_OS_VIP') + elif os.path.isfile('local.yaml'): + with open('local.yaml', 'rb') as f: + self.cfg = yaml.safe_load(f.read()) + + self.vip = self.cfg.get('vip') + else: + amulet.raise_status(amulet.SKIP, + ("please set the vip in local.yaml or env var " + "AMULET_OS_VIP to run this test suite")) + + def _add_services(self): + """Add services + + Add the services that we're testing, where percona-cluster is local, + and the rest of the service are from lp branches that are + compatible with the local charm (e.g. stable or next). + """ + this_service = {'name': 'percona-cluster', + 'units': self.units} + other_services = [{'name': 'hacluster'}] + super(BasicDeployment, self)._add_services(this_service, + other_services) + + def _add_relations(self): + """Add all of the relations for the services.""" + relations = {'percona-cluster:ha': 'hacluster:ha'} + super(BasicDeployment, self)._add_relations(relations) + + def _configure_services(self): + """Configure all of the services.""" + cfg_percona = {'sst-password': 'ubuntu', + 'root-password': 't00r', + 'dataset-size': '512M', + 'vip': self.vip} + + cfg_ha = {'debug': True, + 'corosync_mcastaddr': '226.94.1.4', + 'corosync_key': ('xZP7GDWV0e8Qs0GxWThXirNNYlScgi3sRTdZk/IXKD' + 'qkNFcwdCWfRQnqrHU/6mb6sz6OIoZzX2MtfMQIDcXu' + 'PqQyvKuv7YbRyGHmQwAWDUA4ed759VWAO39kHkfWp9' + 'y5RRk/wcHakTcWYMwm70upDGJEP00YT3xem3NQy27A' + 'C1w=')} + + configs = {'percona-cluster': cfg_percona, + 'hacluster': cfg_ha} + super(BasicDeployment, self)._configure_services(configs) + + def run(self): + # The number of seconds to wait for the environment to setup. + seconds = 1200 + + self._add_services() + self._add_relations() + self._configure_services() + self._deploy() + + i = 0 + while i < 30 and not self.master_unit: + self.master_unit = self.find_master() + i += 1 + time.sleep(10) + + assert self.master_unit is not None, 'percona-cluster vip not found' + + output, code = self.master_unit.run('sudo crm_verify --live-check') + assert code == 0, "'crm_verify --live-check' failed" + + resources = ['res_mysql_vip'] + resources += ['res_mysql_monitor:%d' % i for i in range(self.units)] + + assert sorted(self.get_pcmkr_resources()) == sorted(resources) + + for i in range(self.units): + uid = 'percona-cluster/%d' % i + unit = self.d.sentry.unit[uid] + assert self.is_mysqld_running(unit), 'mysql not running: %s' % uid + + def find_master(self): + for unit_id, unit in self.d.sentry.unit.items(): + if not unit_id.startswith('percona-cluster/'): + continue + + # is the vip running here? + output, code = unit.run('sudo ip a | grep "inet %s/"' % self.vip) + print('---') + print(unit_id) + print(output) + if code == 0: + print('vip(%s) running in %s' % (self.vip, unit_id)) + return unit + + def get_pcmkr_resources(self, unit=None): + if unit: + u = unit + else: + u = self.master_unit + + output, code = u.run('sudo crm_resource -l') + + assert code == 0, 'could not get "crm resource list"' + + return output.split('\n') + + def is_mysqld_running(self, unit=None): + if unit: + u = unit + else: + u = self.master_unit + + output, code = u.run('pidof mysqld') + + if code != 0: + return False + + return self.is_port_open(u, '3306') + + def is_port_open(self, unit=None, port='3306', address=None): + if unit: + addr = unit.info['public-address'] + elif address: + addr = address + else: + raise Exception('Please provide a unit or address') + try: + telnetlib.Telnet(addr, port) + return True + except TimeoutError: # noqa this exception only available in py3 + return False diff --git a/tests/charmhelpers/__init__.py b/tests/charmhelpers/__init__.py new file mode 100644 index 0000000..f72e7f8 --- /dev/null +++ b/tests/charmhelpers/__init__.py @@ -0,0 +1,38 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +# Bootstrap charm-helpers, installing its dependencies if necessary using +# only standard libraries. +import subprocess +import sys + +try: + import six # flake8: noqa +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-six']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-six']) + import six # flake8: noqa + +try: + import yaml # flake8: noqa +except ImportError: + if sys.version_info.major == 2: + subprocess.check_call(['apt-get', 'install', '-y', 'python-yaml']) + else: + subprocess.check_call(['apt-get', 'install', '-y', 'python3-yaml']) + import yaml # flake8: noqa diff --git a/tests/charmhelpers/contrib/__init__.py b/tests/charmhelpers/contrib/__init__.py new file mode 100644 index 0000000..d1400a0 --- /dev/null +++ b/tests/charmhelpers/contrib/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/__init__.py b/tests/charmhelpers/contrib/amulet/__init__.py new file mode 100644 index 0000000..d1400a0 --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py new file mode 100644 index 0000000..367d6b4 --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -0,0 +1,93 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import amulet +import os +import six + + +class AmuletDeployment(object): + """Amulet deployment. + + This class provides generic Amulet deployment and test runner + methods. + """ + + def __init__(self, series=None): + """Initialize the deployment environment.""" + self.series = None + + if series: + self.series = series + self.d = amulet.Deployment(series=self.series) + else: + self.d = amulet.Deployment() + + def _add_services(self, this_service, other_services): + """Add services. + + Add services to the deployment where this_service is the local charm + that we're testing and other_services are the other services that + are being used in the local amulet tests. + """ + if this_service['name'] != os.path.basename(os.getcwd()): + s = this_service['name'] + msg = "The charm's root directory name needs to be {}".format(s) + amulet.raise_status(amulet.FAIL, msg=msg) + + if 'units' not in this_service: + this_service['units'] = 1 + + self.d.add(this_service['name'], units=this_service['units']) + + for svc in other_services: + if 'location' in svc: + branch_location = svc['location'] + elif self.series: + branch_location = 'cs:{}/{}'.format(self.series, svc['name']), + else: + branch_location = None + + if 'units' not in svc: + svc['units'] = 1 + + self.d.add(svc['name'], charm=branch_location, units=svc['units']) + + def _add_relations(self, relations): + """Add all of the relations for the services.""" + for k, v in six.iteritems(relations): + self.d.relate(k, v) + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in six.iteritems(configs): + self.d.configure(service, config) + + def _deploy(self): + """Deploy environment and wait for all hooks to finish executing.""" + try: + self.d.setup(timeout=900) + self.d.sentry.wait(timeout=900) + except amulet.helpers.TimeoutError: + amulet.raise_status(amulet.FAIL, msg="Deployment timed out") + except Exception: + raise + + def run_tests(self): + """Run all of the methods that are prefixed with 'test_'.""" + for test in dir(self): + if test.startswith('test_'): + getattr(self, test)() diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py new file mode 100644 index 0000000..5088b1d --- /dev/null +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -0,0 +1,316 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import ConfigParser +import io +import logging +import re +import sys +import time + +import six + + +class AmuletUtils(object): + """Amulet utilities. + + This class provides common utility functions that are used by Amulet + tests. + """ + + def __init__(self, log_level=logging.ERROR): + self.log = self.get_logger(level=log_level) + + def get_logger(self, name="amulet-logger", level=logging.DEBUG): + """Get a logger object that will log to stdout.""" + log = logging + logger = log.getLogger(name) + fmt = log.Formatter("%(asctime)s %(funcName)s " + "%(levelname)s: %(message)s") + + handler = log.StreamHandler(stream=sys.stdout) + handler.setLevel(level) + handler.setFormatter(fmt) + + logger.addHandler(handler) + logger.setLevel(level) + + return logger + + def valid_ip(self, ip): + if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip): + return True + else: + return False + + def valid_url(self, url): + p = re.compile( + r'^(?:http|ftp)s?://' + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa + r'localhost|' + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' + r'(?::\d+)?' + r'(?:/?|[/?]\S+)$', + re.IGNORECASE) + if p.match(url): + return True + else: + return False + + def validate_services(self, commands): + """Validate services. + + Verify the specified services are running on the corresponding + service units. + """ + for k, v in six.iteritems(commands): + for cmd in v: + output, code = k.run(cmd) + if code != 0: + return "command `{}` returned {}".format(cmd, str(code)) + return None + + def _get_config(self, unit, filename): + """Get a ConfigParser object for parsing a unit's config file.""" + file_contents = unit.file_contents(filename) + config = ConfigParser.ConfigParser() + config.readfp(io.StringIO(file_contents)) + return config + + def validate_config_data(self, sentry_unit, config_file, section, + expected): + """Validate config file data. + + Verify that the specified section of the config file contains + the expected option key:value pairs. + """ + config = self._get_config(sentry_unit, config_file) + + if section != 'DEFAULT' and not config.has_section(section): + return "section [{}] does not exist".format(section) + + for k in expected.keys(): + if not config.has_option(section, k): + return "section [{}] is missing option {}".format(section, k) + if config.get(section, k) != expected[k]: + return "section [{}] {}:{} != expected {}:{}".format( + section, k, config.get(section, k), k, expected[k]) + return None + + def _validate_dict_data(self, expected, actual): + """Validate dictionary data. + + Compare expected dictionary data vs actual dictionary data. + The values in the 'expected' dictionary can be strings, bools, ints, + longs, or can be a function that evaluate a variable and returns a + bool. + """ + self.log.debug('actual: {}'.format(repr(actual))) + self.log.debug('expected: {}'.format(repr(expected))) + + for k, v in six.iteritems(expected): + if k in actual: + if (isinstance(v, six.string_types) or + isinstance(v, bool) or + isinstance(v, six.integer_types)): + if v != actual[k]: + return "{}:{}".format(k, actual[k]) + elif not v(actual[k]): + return "{}:{}".format(k, actual[k]) + else: + return "key '{}' does not exist".format(k) + return None + + def validate_relation_data(self, sentry_unit, relation, expected): + """Validate actual relation data based on expected relation data.""" + actual = sentry_unit.relation(relation[0], relation[1]) + return self._validate_dict_data(expected, actual) + + def _validate_list_data(self, expected, actual): + """Compare expected list vs actual list data.""" + for e in expected: + if e not in actual: + return "expected item {} not found in actual list".format(e) + return None + + def not_null(self, string): + if string is not None: + return True + else: + return False + + def _get_file_mtime(self, sentry_unit, filename): + """Get last modification time of file.""" + return sentry_unit.file_stat(filename)['mtime'] + + def _get_dir_mtime(self, sentry_unit, directory): + """Get last modification time of directory.""" + return sentry_unit.directory_stat(directory)['mtime'] + + def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False): + """Get process' start time. + + Determine start time of the process based on the last modification + time of the /proc/pid directory. If pgrep_full is True, the process + name is matched against the full command line. + """ + if pgrep_full: + cmd = 'pgrep -o -f {}'.format(service) + else: + cmd = 'pgrep -o {}'.format(service) + cmd = cmd + ' | grep -v pgrep || exit 0' + cmd_out = sentry_unit.run(cmd) + self.log.debug('CMDout: ' + str(cmd_out)) + if cmd_out[0]: + self.log.debug('Pid for %s %s' % (service, str(cmd_out[0]))) + proc_dir = '/proc/{}'.format(cmd_out[0].strip()) + return self._get_dir_mtime(sentry_unit, proc_dir) + + def service_restarted(self, sentry_unit, service, filename, + pgrep_full=False, sleep_time=20): + """Check if service was restarted. + + Compare a service's start time vs a file's last modification time + (such as a config file for that service) to determine if the service + has been restarted. + """ + time.sleep(sleep_time) + if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >= + self._get_file_mtime(sentry_unit, filename)): + return True + else: + return False + + def service_restarted_since(self, sentry_unit, mtime, service, + pgrep_full=False, sleep_time=20, + retry_count=2): + """Check if service was been started after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Returns: + bool: True if service found and its start time it newer than mtime, + False if service is older than mtime or if service was + not found. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + while retry_count > 0 and not proc_start_time: + self.log.debug('No pid file found for service %s, will retry %i ' + 'more times' % (service, retry_count)) + time.sleep(30) + proc_start_time = self._get_proc_start_time(sentry_unit, service, + pgrep_full) + retry_count = retry_count - 1 + + if not proc_start_time: + self.log.warn('No proc start time found, assuming service did ' + 'not start') + return False + if proc_start_time >= mtime: + self.log.debug('proc start time is newer than provided mtime' + '(%s >= %s)' % (proc_start_time, mtime)) + return True + else: + self.log.warn('proc start time (%s) is older than provided mtime ' + '(%s), service did not restart' % (proc_start_time, + mtime)) + return False + + def config_updated_since(self, sentry_unit, filename, mtime, + sleep_time=20): + """Check if file was modified after a given time. + + Args: + sentry_unit (sentry): The sentry unit to check the file mtime on + filename (string): The file to check mtime of + mtime (float): The epoch time to check against + sleep_time (int): Seconds to sleep before looking for process + + Returns: + bool: True if file was modified more recently than mtime, False if + file was modified before mtime, + """ + self.log.debug('Checking %s updated since %s' % (filename, mtime)) + time.sleep(sleep_time) + file_mtime = self._get_file_mtime(sentry_unit, filename) + if file_mtime >= mtime: + self.log.debug('File mtime is newer than provided mtime ' + '(%s >= %s)' % (file_mtime, mtime)) + return True + else: + self.log.warn('File mtime %s is older than provided mtime %s' + % (file_mtime, mtime)) + return False + + def validate_service_config_changed(self, sentry_unit, mtime, service, + filename, pgrep_full=False, + sleep_time=20, retry_count=2): + """Check service and file were updated after mtime + + Args: + sentry_unit (sentry): The sentry unit to check for the service on + mtime (float): The epoch time to check against + service (string): service name to look for in process table + filename (string): The file to check mtime of + pgrep_full (boolean): Use full command line search mode with pgrep + sleep_time (int): Seconds to sleep before looking for process + retry_count (int): If service is not found, how many times to retry + + Typical Usage: + u = OpenStackAmuletUtils(ERROR) + ... + mtime = u.get_sentry_time(self.cinder_sentry) + self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'}) + if not u.validate_service_config_changed(self.cinder_sentry, + mtime, + 'cinder-api', + '/etc/cinder/cinder.conf') + amulet.raise_status(amulet.FAIL, msg='update failed') + Returns: + bool: True if both service and file where updated/restarted after + mtime, False if service is older than mtime or if service was + not found or if filename was modified before mtime. + """ + self.log.debug('Checking %s restarted since %s' % (service, mtime)) + time.sleep(sleep_time) + service_restart = self.service_restarted_since(sentry_unit, mtime, + service, + pgrep_full=pgrep_full, + sleep_time=0, + retry_count=retry_count) + config_update = self.config_updated_since(sentry_unit, filename, mtime, + sleep_time=0) + return service_restart and config_update + + def get_sentry_time(self, sentry_unit): + """Return current epoch time on a sentry""" + cmd = "date +'%s'" + return float(sentry_unit.run(cmd)[0]) + + def relation_error(self, name, data): + return 'unexpected relation data in {} - {}'.format(name, data) + + def endpoint_error(self, name, data): + return 'unexpected endpoint data in {} - {}'.format(name, data) diff --git a/tests/charmhelpers/contrib/openstack/__init__.py b/tests/charmhelpers/contrib/openstack/__init__.py new file mode 100644 index 0000000..d1400a0 --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/__init__.py b/tests/charmhelpers/contrib/openstack/amulet/__init__.py new file mode 100644 index 0000000..d1400a0 --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py new file mode 100644 index 0000000..11d49a7 --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -0,0 +1,137 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import six +from collections import OrderedDict +from charmhelpers.contrib.amulet.deployment import ( + AmuletDeployment +) + + +class OpenStackAmuletDeployment(AmuletDeployment): + """OpenStack amulet deployment. + + This class inherits from AmuletDeployment and has additional support + that is specifically for use by OpenStack charms. + """ + + def __init__(self, series=None, openstack=None, source=None, stable=True): + """Initialize the deployment environment.""" + super(OpenStackAmuletDeployment, self).__init__(series) + self.openstack = openstack + self.source = source + self.stable = stable + # Note(coreycb): this needs to be changed when new next branches come + # out. + self.current_next = "trusty" + + def _determine_branch_locations(self, other_services): + """Determine the branch locations for the other services. + + Determine if the local branch being tested is derived from its + stable or next (dev) branch, and based on this, use the corresonding + stable or next branches for the other_services.""" + base_charms = ['mysql', 'mongodb'] + + if self.stable: + for svc in other_services: + temp = 'lp:charms/{}' + svc['location'] = temp.format(svc['name']) + else: + for svc in other_services: + if svc['name'] in base_charms: + temp = 'lp:charms/{}' + svc['location'] = temp.format(svc['name']) + else: + temp = 'lp:~openstack-charmers/charms/{}/{}/next' + svc['location'] = temp.format(self.current_next, + svc['name']) + return other_services + + def _add_services(self, this_service, other_services): + """Add services to the deployment and set openstack-origin/source.""" + other_services = self._determine_branch_locations(other_services) + + super(OpenStackAmuletDeployment, self)._add_services(this_service, + other_services) + + services = other_services + services.append(this_service) + use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph', + 'ceph-osd', 'ceph-radosgw'] + # Openstack subordinate charms do not expose an origin option as that + # is controlled by the principle + ignore = ['neutron-openvswitch'] + + if self.openstack: + for svc in services: + if svc['name'] not in use_source + ignore: + config = {'openstack-origin': self.openstack} + self.d.configure(svc['name'], config) + + if self.source: + for svc in services: + if svc['name'] in use_source and svc['name'] not in ignore: + config = {'source': self.source} + self.d.configure(svc['name'], config) + + def _configure_services(self, configs): + """Configure all of the services.""" + for service, config in six.iteritems(configs): + self.d.configure(service, config) + + def _get_openstack_release(self): + """Get openstack release. + + Return an integer representing the enum value of the openstack + release. + """ + (self.precise_essex, self.precise_folsom, self.precise_grizzly, + self.precise_havana, self.precise_icehouse, + self.trusty_icehouse, self.trusty_juno, self.trusty_kilo, + self.utopic_juno, self.vivid_kilo) = range(10) + releases = { + ('precise', None): self.precise_essex, + ('precise', 'cloud:precise-folsom'): self.precise_folsom, + ('precise', 'cloud:precise-grizzly'): self.precise_grizzly, + ('precise', 'cloud:precise-havana'): self.precise_havana, + ('precise', 'cloud:precise-icehouse'): self.precise_icehouse, + ('trusty', None): self.trusty_icehouse, + ('trusty', 'cloud:trusty-juno'): self.trusty_juno, + ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo, + ('utopic', None): self.utopic_juno, + ('vivid', None): self.vivid_kilo} + return releases[(self.series, self.openstack)] + + def _get_openstack_release_string(self): + """Get openstack release string. + + Return a string representing the openstack release. + """ + releases = OrderedDict([ + ('precise', 'essex'), + ('quantal', 'folsom'), + ('raring', 'grizzly'), + ('saucy', 'havana'), + ('trusty', 'icehouse'), + ('utopic', 'juno'), + ('vivid', 'kilo'), + ]) + if self.openstack: + os_origin = self.openstack.split(':')[1] + return os_origin.split('%s-' % self.series)[1].split('/')[0] + else: + return releases[self.series] diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py new file mode 100644 index 0000000..9c3d918 --- /dev/null +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -0,0 +1,294 @@ +# Copyright 2014-2015 Canonical Limited. +# +# This file is part of charm-helpers. +# +# charm-helpers is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 as +# published by the Free Software Foundation. +# +# charm-helpers is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with charm-helpers. If not, see . + +import logging +import os +import time +import urllib + +import glanceclient.v1.client as glance_client +import keystoneclient.v2_0 as keystone_client +import novaclient.v1_1.client as nova_client + +import six + +from charmhelpers.contrib.amulet.utils import ( + AmuletUtils +) + +DEBUG = logging.DEBUG +ERROR = logging.ERROR + + +class OpenStackAmuletUtils(AmuletUtils): + """OpenStack amulet utilities. + + This class inherits from AmuletUtils and has additional support + that is specifically for use by OpenStack charms. + """ + + def __init__(self, log_level=ERROR): + """Initialize the deployment environment.""" + super(OpenStackAmuletUtils, self).__init__(log_level) + + def validate_endpoint_data(self, endpoints, admin_port, internal_port, + public_port, expected): + """Validate endpoint data. + + Validate actual endpoint data vs expected endpoint data. The ports + are used to find the matching endpoint. + """ + found = False + for ep in endpoints: + self.log.debug('endpoint: {}'.format(repr(ep))) + if (admin_port in ep.adminurl and + internal_port in ep.internalurl and + public_port in ep.publicurl): + found = True + actual = {'id': ep.id, + 'region': ep.region, + 'adminurl': ep.adminurl, + 'internalurl': ep.internalurl, + 'publicurl': ep.publicurl, + 'service_id': ep.service_id} + ret = self._validate_dict_data(expected, actual) + if ret: + return 'unexpected endpoint data - {}'.format(ret) + + if not found: + return 'endpoint not found' + + def validate_svc_catalog_endpoint_data(self, expected, actual): + """Validate service catalog endpoint data. + + Validate a list of actual service catalog endpoints vs a list of + expected service catalog endpoints. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for k, v in six.iteritems(expected): + if k in actual: + ret = self._validate_dict_data(expected[k][0], actual[k][0]) + if ret: + return self.endpoint_error(k, ret) + else: + return "endpoint {} does not exist".format(k) + return ret + + def validate_tenant_data(self, expected, actual): + """Validate tenant data. + + Validate a list of actual tenant data vs list of expected tenant + data. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'description': act.description, + 'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected tenant data - {}".format(ret) + if not found: + return "tenant {} does not exist".format(e['name']) + return ret + + def validate_role_data(self, expected, actual): + """Validate role data. + + Validate a list of actual role data vs a list of expected role + data. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'name': act.name, 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected role data - {}".format(ret) + if not found: + return "role {} does not exist".format(e['name']) + return ret + + def validate_user_data(self, expected, actual): + """Validate user data. + + Validate a list of actual user data vs a list of expected user + data. + """ + self.log.debug('actual: {}'.format(repr(actual))) + for e in expected: + found = False + for act in actual: + a = {'enabled': act.enabled, 'name': act.name, + 'email': act.email, 'tenantId': act.tenantId, + 'id': act.id} + if e['name'] == a['name']: + found = True + ret = self._validate_dict_data(e, a) + if ret: + return "unexpected user data - {}".format(ret) + if not found: + return "user {} does not exist".format(e['name']) + return ret + + def validate_flavor_data(self, expected, actual): + """Validate flavor data. + + Validate a list of actual flavors vs a list of expected flavors. + """ + self.log.debug('actual: {}'.format(repr(actual))) + act = [a.name for a in actual] + return self._validate_list_data(expected, act) + + def tenant_exists(self, keystone, tenant): + """Return True if tenant exists.""" + return tenant in [t.name for t in keystone.tenants.list()] + + def authenticate_keystone_admin(self, keystone_sentry, user, password, + tenant): + """Authenticates admin user with the keystone admin endpoint.""" + unit = keystone_sentry + service_ip = unit.relation('shared-db', + 'mysql:shared-db')['private-address'] + ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8')) + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_keystone_user(self, keystone, user, password, tenant): + """Authenticates a regular user with the keystone public endpoint.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return keystone_client.Client(username=user, password=password, + tenant_name=tenant, auth_url=ep) + + def authenticate_glance_admin(self, keystone): + """Authenticates admin user with glance.""" + ep = keystone.service_catalog.url_for(service_type='image', + endpoint_type='adminURL') + return glance_client.Client(ep, token=keystone.auth_token) + + def authenticate_nova_user(self, keystone, user, password, tenant): + """Authenticates a regular user with nova-api.""" + ep = keystone.service_catalog.url_for(service_type='identity', + endpoint_type='publicURL') + return nova_client.Client(username=user, api_key=password, + project_id=tenant, auth_url=ep) + + def create_cirros_image(self, glance, image_name): + """Download the latest cirros image and upload it to glance.""" + http_proxy = os.getenv('AMULET_HTTP_PROXY') + self.log.debug('AMULET_HTTP_PROXY: {}'.format(http_proxy)) + if http_proxy: + proxies = {'http': http_proxy} + opener = urllib.FancyURLopener(proxies) + else: + opener = urllib.FancyURLopener() + + f = opener.open("http://download.cirros-cloud.net/version/released") + version = f.read().strip() + cirros_img = "cirros-{}-x86_64-disk.img".format(version) + local_path = os.path.join('tests', cirros_img) + + if not os.path.exists(local_path): + cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", + version, cirros_img) + opener.retrieve(cirros_url, local_path) + f.close() + + with open(local_path) as f: + image = glance.images.create(name=image_name, is_public=True, + disk_format='qcow2', + container_format='bare', data=f) + count = 1 + status = image.status + while status != 'active' and count < 10: + time.sleep(3) + image = glance.images.get(image.id) + status = image.status + self.log.debug('image status: {}'.format(status)) + count += 1 + + if status != 'active': + self.log.error('image creation timed out') + return None + + return image + + def delete_image(self, glance, image): + """Delete the specified image.""" + num_before = len(list(glance.images.list())) + glance.images.delete(image) + + count = 1 + num_after = len(list(glance.images.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(glance.images.list())) + self.log.debug('number of images: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('image deletion timed out') + return False + + return True + + def create_instance(self, nova, image_name, instance_name, flavor): + """Create the specified instance.""" + image = nova.images.find(name=image_name) + flavor = nova.flavors.find(name=flavor) + instance = nova.servers.create(name=instance_name, image=image, + flavor=flavor) + + count = 1 + status = instance.status + while status != 'ACTIVE' and count < 60: + time.sleep(3) + instance = nova.servers.get(instance.id) + status = instance.status + self.log.debug('instance status: {}'.format(status)) + count += 1 + + if status != 'ACTIVE': + self.log.error('instance creation timed out') + return None + + return instance + + def delete_instance(self, nova, instance): + """Delete the specified instance.""" + num_before = len(list(nova.servers.list())) + nova.servers.delete(instance) + + count = 1 + num_after = len(list(nova.servers.list())) + while num_after != (num_before - 1) and count < 10: + time.sleep(3) + num_after = len(list(nova.servers.list())) + self.log.debug('number of instances: {}'.format(num_after)) + count += 1 + + if num_after != (num_before - 1): + self.log.error('instance deletion timed out') + return False + + return True diff --git a/unit_tests/test_percona_hooks.py b/unit_tests/test_percona_hooks.py new file mode 100644 index 0000000..65d3059 --- /dev/null +++ b/unit_tests/test_percona_hooks.py @@ -0,0 +1,65 @@ +import mock +import sys +from test_utils import CharmTestCase + +sys.modules['MySQLdb'] = mock.Mock() +import percona_hooks as hooks + +TO_PATCH = ['log', 'config', + 'get_db_helper', + 'relation_ids', + 'relation_set'] + + +class TestHaRelation(CharmTestCase): + def setUp(self): + CharmTestCase.setUp(self, hooks, TO_PATCH) + + @mock.patch('sys.exit') + def test_relation_not_configured(self, exit_): + self.config.return_value = None + + class MyError(Exception): + pass + + def f(x): + raise MyError(x) + exit_.side_effect = f + self.assertRaises(MyError, hooks.ha_relation_joined) + + def test_resources(self): + self.relation_ids.return_value = ['ha:1'] + password = 'ubuntu' + helper = mock.Mock() + attrs = {'get_mysql_password.return_value': password} + helper.configure_mock(**attrs) + self.get_db_helper.return_value = helper + self.test_config.set('vip', '10.0.3.3') + self.test_config.set('sst-password', password) + def f(k): + return self.test_config.get(k) + + self.config.side_effect = f + hooks.ha_relation_joined() + + resources = {'res_mysql_vip': 'ocf:heartbeat:IPaddr2', + 'res_mysql_monitor': 'ocf:percona:mysql_monitor'} + resource_params = {'res_mysql_vip': ('params ip="10.0.3.3" ' + 'cidr_netmask="24" ' + 'nic="eth0"'), + 'res_mysql_monitor': + hooks.RES_MONITOR_PARAMS % {'sstpass': 'ubuntu'}} + groups = {'grp_percona_cluster': 'res_mysql_vip'} + + clones = {'cl_mysql_monitor': 'res_mysql_monitor meta interleave=true'} + + colocations = {'vip_mysqld': 'inf: grp_percona_cluster cl_mysql_monitor'} + + locations = {'loc_percona_cluster': + 'grp_percona_cluster rule inf: writable eq 1'} + + self.relation_set.assert_called_with( + relation_id='ha:1', corosync_bindiface=f('ha-bindiface'), + corosync_mcastport=f('ha-mcastport'), resources=resources, + resource_params=resource_params, groups=groups, + clones=clones, colocations=colocations, locations=locations) diff --git a/unit_tests/test_utils.py b/unit_tests/test_utils.py new file mode 100644 index 0000000..a59f897 --- /dev/null +++ b/unit_tests/test_utils.py @@ -0,0 +1,121 @@ +import logging +import unittest +import os +import yaml + +from contextlib import contextmanager +from mock import patch, MagicMock + + +def load_config(): + ''' + Walk backwords from __file__ looking for config.yaml, load and return the + 'options' section' + ''' + config = None + f = __file__ + while config is None: + d = os.path.dirname(f) + if os.path.isfile(os.path.join(d, 'config.yaml')): + config = os.path.join(d, 'config.yaml') + break + f = d + + if not config: + logging.error('Could not find config.yaml in any parent directory ' + 'of %s. ' % file) + raise Exception + + return yaml.safe_load(open(config).read())['options'] + + +def get_default_config(): + ''' + Load default charm config from config.yaml return as a dict. + If no default is set in config.yaml, its value is None. + ''' + default_config = {} + config = load_config() + for k, v in config.iteritems(): + if 'default' in v: + default_config[k] = v['default'] + else: + default_config[k] = None + return default_config + + +class CharmTestCase(unittest.TestCase): + + def setUp(self, obj, patches): + super(CharmTestCase, self).setUp() + self.patches = patches + self.obj = obj + self.test_config = TestConfig() + self.test_relation = TestRelation() + self.patch_all() + + def patch(self, method): + _m = patch.object(self.obj, method) + mock = _m.start() + self.addCleanup(_m.stop) + return mock + + def patch_all(self): + for method in self.patches: + setattr(self, method, self.patch(method)) + + +class TestConfig(object): + + def __init__(self): + self.config = get_default_config() + + def get(self, attr=None): + if not attr: + return self.get_all() + try: + return self.config[attr] + except KeyError: + return None + + def get_all(self): + return self.config + + def set(self, attr, value): + if attr not in self.config: + raise KeyError + self.config[attr] = value + + +class TestRelation(object): + + def __init__(self, relation_data={}): + self.relation_data = relation_data + + def set(self, relation_data): + self.relation_data = relation_data + + def get(self, attr=None, unit=None, rid=None): + if attr is None: + return self.relation_data + elif attr in self.relation_data: + return self.relation_data[attr] + return None + + +@contextmanager +def patch_open(): + '''Patch open() to allow mocking both open() itself and the file that is + yielded. + + Yields the mock for "open" and "file", respectively.''' + mock_open = MagicMock(spec=open) + mock_file = MagicMock(spec=file) + + @contextmanager + def stub_open(*args, **kwargs): + mock_open(*args, **kwargs) + yield mock_file + + with patch('__builtin__.open', stub_open): + yield mock_open, mock_file