Improve Galera configuration for clustering

Debian only.

The problem here is that the first time the galera is started it must be
done with some specific option, ie "--wsrep-new-cluster".  Then it
shouldn't be used anymore.  This has to be used only on one node or else
all nodes become master.

To achieve this behaviour a specific mysql init script is installed:
mysql-bootstrap.

This script check if the galera has started and, if it's not the case,
it adds wsrep-new-cluster to the command line.

Then in puppet I change the service name for mysql to mysql-bootstrap on
one node.

The port to RHEL could take the same road.  The only requirement is
that:

$ strings `which mysqld` | grep wsrep-new-cluster

be true.

I don't add the close 11 to it as the rhel support should come with it I
think.

issue #11

manifests/database/sql.pp

@@ -15,12 +15,15 @@
 #
 # MySQL Galera Node
 #
-
+# === Parameters
+#
+#  [*galera_internal_ips*]
+#    Array of internal ip of the galera nodes.
 class cloud::database::sql (
     $api_eth                        = $os_params::api_eth,
     $service_provider               = 'sysv',
-    $galera_nextserver              = $os_params::galera_nextserver,
     $galera_master_name             = $os_params::galera_master_name,
+    $galera_internal_ips            = $os_params::galera_internal_ips,
     $keystone_db_host               = $os_params::keystone_db_host,
     $keystone_db_user               = $os_params::keystone_db_user,
     $keystone_db_password           = $os_params::keystone_db_password,
@@ -90,16 +93,26 @@ class cloud::database::sql (
     }
   }
 
-  class { 'mysql::server':
+  if($::osfamily == 'Debian'){
-    config_hash         => {
+    class { 'cloud::debian::galera_patch' : }
-      bind_address      => $api_eth,
-      root_password     => $mysql_root_password,
-    },
-    notify              => Service['xinetd'],
   }
 
+  $gcomm_base = inline_template('<%= @galera_internal_ips.join(",") + "?pc.wait_prim=no" -%>')
+
   if $::hostname == $galera_master_name {
 
+
+    class { 'mysql::server':
+      config_hash         => {
+        bind_address      => $api_eth,
+        root_password     => $mysql_root_password,
+        service_name      => 'mysql-bootstrap',
+      },
+      notify              => Service['xinetd'],
+    }
+
+    $gcomm_definition = "${gcomm_base}&pc.bootstrap=1"
+
 # OpenStack DB
     class { 'keystone::db::mysql':
       dbname        => 'keystone',
@@ -175,8 +188,18 @@ class cloud::database::sql (
     }
 
     Database_user<<| |>>
+  } else {
+    $gcomm_definition = $gcomm_base
 
-  } # if $::hostname == $galera_master_name
+    class { 'mysql::server':
+      config_hash         => {
+        bind_address      => $api_eth,
+        root_password     => $mysql_root_password,
+        service_name      => 'mysql',
+      },
+      notify              => Service['xinetd'],
+    }
+  } # if $::hostname == $galera_master
 
   # Haproxy http monitoring
   file_line { 'mysqlchk-in-etc-services':

manifests/debian/galera_patch.pp

@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2013 eNovance SAS <licensing@enovance.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: cloud::debian::galera_patch
+#
+# Install a dedicated mysqld init script
+#
+# This is due to this bug: https://bugs.launchpad.net/codership-mysql/+bug/1087368
+#
+# The backport to API 23 requires a command line option --wsrep-new-cluster:
+#
+# http://bazaar.launchpad.net/~codership/codership-mysql/wsrep-5.5/revision/3844?start_revid=3844
+#
+# and the mysql init script cannot have arguments passed to the daemon
+# using /etc/default/mysql standart mechanism
+#
+# To check that the mysqld support the options you can :
+# strings `which mysqld` | grep wsrep-new-cluster
+#
+# TODO: to be remove as soon as the API 25 is packaged, ie galera 3 ...
+class cloud::debian::galera_patch (
+  ) {
+
+  # replace the file
+  file { '/etc/init.d/mysql-bootstrap':
+    content => template('cloud/database/etc_initd_mysql'),
+    owner   => 'root',
+    mode    => '0755',
+    group   => 'root',
+    notify  => Service['mysqld'],
+    before  => Package['mysql-server'],
+  }
+}

templates/database/etc_initd_mysql

@@ -0,0 +1,200 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:          mysql
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Should-Start:      $network $named $time
+# Should-Stop:       $network $named $time
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start and stop the mysql database server daemon
+# Description:       Controls the main MariaDB database server daemon "mysqld"
+#                    and its wrapper script "mysqld_safe".
+### END INIT INFO
+#
+MYSQLD_STARTUP_TIMEOUT=${MYSQLD_STARTUP_TIMEOUT:-60}
+[ -e /etc/mysql/my.cnf ] && \
+    MYSQLD_DATA_DIR=$(awk -F= '/^datadir/{print $2}' /etc/mysql/my.cnf | sed -e 's/^ *//')
+MYSQLD_DATA_DIR=${MYSQLD_DATA_DIR:-<%= scope.lookupvar('::mysql::datadir') %>}
+set -e
+set -u
+${DEBIAN_SCRIPT_DEBUG:+ set -v -x}
+
+test -x /usr/sbin/mysqld || exit 0
+
+. /lib/lsb/init-functions
+
+SELF=$(cd $(dirname $0); pwd -P)/$(basename $0)
+CONF=/etc/mysql/my.cnf
+MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
+
+# priority can be overriden and "-s" adds output to stderr
+ERR_LOGGER="logger -p daemon.err -t /etc/init.d/mysql -i"
+
+# Safeguard (relative paths, core dumps..)
+cd /
+umask 077
+
+# mysqladmin likes to read /root/.my.cnf. This is usually not what I want
+# as many admins e.g. only store a password without a username there and
+# so break my scripts.
+export HOME=/etc/mysql/
+
+## Fetch a particular option from mysql's invocation.
+#
+# Usage: void mysqld_get_param option
+mysqld_get_param() {
+	/usr/sbin/mysqld --print-defaults \
+		| tr " " "\n" \
+		| grep -- "--$1" \
+		| tail -n 1 \
+		| cut -d= -f2
+}
+
+## Do some sanity checks before even trying to start mysqld.
+sanity_checks() {
+  # check for config file
+  if [ ! -r /etc/mysql/my.cnf ]; then
+    log_warning_msg "$0: WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz"
+    echo                "WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz" | $ERR_LOGGER
+  fi
+
+  # check for diskspace shortage
+  datadir=`mysqld_get_param datadir`
+  if LC_ALL=C BLOCKSIZE= df --portability $datadir/. | tail -n 1 | awk '{ exit ($4>4096) }'; then
+    log_failure_msg "$0: ERROR: The partition with $datadir is too full!"
+    echo                "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
+    exit 1
+  fi
+}
+
+## Checks if there is a server running and if so if it is accessible.
+#
+# check_alive insists on a pingable server
+# check_dead also fails if there is a lost mysqld in the process list
+#
+# Usage: boolean mysqld_status [check_alive|check_dead] [warn|nowarn]
+mysqld_status () {
+    ping_output=`$MYADMIN ping 2>&1`; ping_alive=$(( ! $? ))
+
+    ps_alive=0
+    pidfile=`mysqld_get_param pid-file`
+    if [ -f "$pidfile" ] && ps `cat $pidfile` >/dev/null 2>&1; then ps_alive=1; fi
+    
+    if [ "$1" = "check_alive"  -a  $ping_alive = 1 ] ||
+       [ "$1" = "check_dead"   -a  $ping_alive = 0  -a  $ps_alive = 0 ]; then
+	return 0 # EXIT_SUCCESS
+    else
+  	if [ "$2" = "warn" ]; then
+  	    echo -e "$ps_alive processes alive and '$MYADMIN ping' resulted in\n$ping_output\n" | $ERR_LOGGER -p daemon.debug
+	fi
+  	return 1 # EXIT_FAILURE
+    fi
+}
+
+#
+# main()
+#
+
+case "${1:-''}" in
+  'start')
+	sanity_checks;
+	# Start daemon
+	log_daemon_msg "Starting MariaDB database server" "mysqld"
+	if mysqld_status check_alive nowarn; then
+	   log_progress_msg "already running"
+	   log_end_msg 0
+	else
+	    # Could be removed during boot
+	    test -e /var/run/mysqld || install -m 755 -o mysql -g root -d /var/run/mysqld
+
+	    # Start MariaDB!  in a Galera setup we want to use
+	    # new-cluster only when the galera cluster hasn't been
+	    # bootstraped
+	    if [ -e ${MYSQLD_DATA_DIR}/grastate.dat ]; then
+	        # normal boot
+	        /usr/bin/mysqld_safe "${@:2}"  > /dev/null 2>&1 &
+	    else
+	        # bootstrap boot
+	        log_progress_msg " (Galera bootstrap) "
+	        /usr/bin/mysqld_safe "${@:2}" --wsrep-new-cluster > /dev/null 2>&1 &
+	    fi
+
+	    # 6s was reported in #352070 to be too few when using ndbcluster
+	    for i in $(seq 1 "${MYSQLD_STARTUP_TIMEOUT:-30}"); do
+                sleep 1
+	        if mysqld_status check_alive nowarn ; then break; fi
+		log_progress_msg "."
+	    done
+	    if mysqld_status check_alive warn; then
+                log_end_msg 0
+	        # Now start mysqlcheck or whatever the admin wants.
+	        output=$(/etc/mysql/debian-start)
+		[ -n "$output" ] && log_action_msg "$output"
+	    else
+	        log_end_msg 1
+		log_failure_msg "Please take a look at the syslog"
+	    fi
+	fi
+	;;
+
+  'stop')
+	# * As a passwordless mysqladmin (e.g. via ~/.my.cnf) must be possible
+	# at least for cron, we can rely on it here, too. (although we have 
+	# to specify it explicit as e.g. sudo environments points to the normal
+	# users home and not /root)
+	log_daemon_msg "Stopping MariaDB database server" "mysqld"
+	if ! mysqld_status check_dead nowarn; then
+	  set +e
+	  shutdown_out=`$MYADMIN shutdown 2>&1`; r=$?
+	  set -e
+	  if [ "$r" -ne 0 ]; then
+	    log_end_msg 1
+	    [ "$VERBOSE" != "no" ] && log_failure_msg "Error: $shutdown_out"
+	    log_daemon_msg "Killing MariaDB database server by signal" "mysqld"
+	    killall -15 mysqld
+            server_down=
+	    for i in `seq 1 600`; do
+              sleep 1
+              if mysqld_status check_dead nowarn; then server_down=1; break; fi
+            done
+          if test -z "$server_down"; then killall -9 mysqld; fi
+	  fi
+        fi
+
+        if ! mysqld_status check_dead warn; then
+	  log_end_msg 1
+	  log_failure_msg "Please stop MariaDB manually and read /usr/share/doc/mariadb-server-5.5/README.Debian.gz!"
+	  exit -1
+	else
+	  log_end_msg 0
+        fi
+	;;
+
+  'restart')
+	set +e; $SELF stop; set -e
+	$SELF start 
+	;;
+
+  'reload'|'force-reload')
+  	log_daemon_msg "Reloading MariaDB database server" "mysqld"
+	$MYADMIN reload
+	log_end_msg 0
+	;;
+
+  'status')
+	if mysqld_status check_alive nowarn; then
+	  log_action_msg "$($MYADMIN version)"
+	else
+	  log_action_msg "MariaDB is stopped."
+	  exit 3
+	fi
+  	;;
+
+  *)
+	echo "Usage: $SELF start|stop|restart|reload|force-reload|status"
+	exit 1
+	;;
+esac
+

templates/database/mysql.conf.erb

@@ -42,14 +42,8 @@ innodb_autoinc_lock_mode        = 2
 innodb_locks_unsafe_for_binlog  = 1
 wsrep_provider                  = "<%= @wsrep_provider %>"
 wsrep_cluster_name              = "galera_cluster"
-<% if @hostname != @galera_master_name -%>
+wsrep_cluster_address           = "gcomm://<%= @gcomm_definition %>"
-# This node is not Galera Master (connect wsrep to galera_master_name)
+wsrep_sst_auth                  = root:<%= @mysql_root_password %>
-wsrep_cluster_address           = "gcomm://<%= @galera_nextserver[0] %>"
-<% else -%>
-# This node is Galera Master (connect wsrep to localhost)
-wsrep_cluster_address           = "gcomm://"
-<% end -%>
-wsrep_sst_auth                  = root:<%= @mysql_password %>
 wsrep_certify_nonPK             = 1
 wsrep_convert_LOCK_to_trx       = 0
 wsrep_auto_increment_control    = 1