Wait for rabbitmq before starting lma_collector

If we start lma_collector before the availability of rabbitmq cluster it
will fail to connect to the lma queues and then, it will fail to start.
It may take several long minutes before pacemaker starts the service.
So we need to be sure that rabbitmq cluster is up and running before
starting lma_collector.

Change-Id: Ia254b744f4173f64ee3ab8200b2896ecc412d06f

deployment_scripts/puppet/manifests/base.pp

@@ -39,9 +39,30 @@ if hiera('deployment_mode') =~ /^ha_/ and hiera('role') =~ /controller/{
   $additional_groups = []
 }
 
+if hiera('role') =~ /controller/ and $lma_collector['enable_notifications'] {
+  $pre_script        = '/usr/local/bin/wait_for_rabbitmq'
+  # Params used by the script.
+  $rabbit            = hiera('rabbit')
+  $rabbitmq_user     = 'nova'
+  $rabbitmq_password = $rabbit['password']
+  $wait_delay        = 30
+
+  file { $pre_script:
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('lma_collector/wait_for_rabbitmq.erb'),
+    before  => Class['lma_collector']
+  }
+} else {
+  $pre_script = undef
+}
+
 class { 'lma_collector':
-  tags   => merge($tags, $additional_tags),
-  groups => $additional_groups,
+  tags       => merge($tags, $additional_tags),
+  groups     => $additional_groups,
+  pre_script => $pre_script,
 }
 
 class { 'lma_collector::logs::system':

deployment_scripts/puppet/modules/heka/manifests/init.pp

@@ -68,11 +68,13 @@ class heka (
   $maxprocs = $heka::params::maxprocs,
   $dashboard_address = $heka::params::dashboard_address,
   $dashboard_port = $heka::params::dashboard_port,
+  $pre_script = undef,
 ) inherits heka::params {
 
-  $heka_user = $heka::params::user
-  $base_dir = "/var/cache/${service_name}"
-  $log_file = "/var/log/${service_name}.log"
+  $heka_user     = $heka::params::user
+  $hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
+  $base_dir      = "/var/cache/${service_name}"
+  $log_file      = "/var/log/${service_name}.log"
 
   package { $heka::params::package_name:
     ensure => present,
@@ -121,6 +123,14 @@ class heka (
     content => template('heka/logrotate.conf.erb'),
   }
 
+  file { $hekad_wrapper:
+    ensure  => present,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0755',
+    content => template('heka/hekad_wrapper.erb'),
+  }
+
   case $::osfamily {
     'Debian': {
       file {"/etc/init/${service_name}.conf":
@@ -128,6 +138,7 @@ class heka (
         content => template('heka/hekad.upstart.conf.erb'),
         notify  => Service[$service_name],
         alias   => 'heka_init_script',
+        require => File[$hekad_wrapper],
       }
     }
 
@@ -138,6 +149,7 @@ class heka (
         mode    => '0755',
         notify  => Service[$service_name],
         alias   => 'heka_init_script',
+        require => File[$hekad_wrapper],
       }
     }
     default: {

deployment_scripts/puppet/modules/heka/templates/hekad.initd.erb

@@ -13,7 +13,7 @@ if [ -f /etc/rc.d/init.d/functions ]; then
     . /etc/rc.d/init.d/functions
 fi
 
-exec="/usr/bin/hekad"
+exec="<%= @hekad_wrapper %>"
 prog="<%= @service_name %>"
 pidfile=/var/run/${prog}.pid
 
@@ -31,7 +31,7 @@ start() {
     chown <%= @heka_user %>:<%= @heka_user %> <%= @log_file %>
 <% end -%>
     echo -n $"Starting $prog: "
-    daemonize -p $pidfile -e <%= @log_file %> <%= @run_as_root ? "" : "-u #{ @heka_user }" %> -l $lockfile $exec -config=<%= @config_dir %>
+    daemonize -p $pidfile -e <%= @log_file %> <%= @run_as_root ? "" : "-u #{ @heka_user }" %> -l $lockfile $exec
     retval=$?
     [ $retval -eq 0 ] && success || failure
     echo
@@ -41,8 +41,9 @@ start() {
 
 stop() {
     echo -n $"Stopping $prog: "
-    killproc -p $pidfile $prog
+    pkill -P $(cat $pidfile)
     retval=$?
+    rm -f $pidfile
     echo
     [ $retval -eq 0 ] && rm -f $lockfile
     return $retval

deployment_scripts/puppet/modules/heka/templates/hekad.upstart.conf.erb

@@ -14,4 +14,4 @@ pre-start script
 end script
 <% end -%>
 
-exec <%= @run_as_root ? "" : "sudo -u #{ @heka_user }" %> /usr/bin/hekad -config=<%= @config_dir %> 2>><%= @log_file %>
+exec <%= @run_as_root ? "" : "sudo -u #{ @heka_user }" %> <%= @hekad_wrapper %> 2>><%= @log_file %>

deployment_scripts/puppet/modules/heka/templates/hekad_wrapper.erb

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+HEKAD="/usr/bin/hekad"
+
+<% if @pre_script -%>
+<%= @pre_script %>
+<% end -%>
+
+$HEKAD -config=<%= @config_dir %>

deployment_scripts/puppet/modules/lma_collector/manifests/init.pp

@@ -32,6 +32,7 @@
 class lma_collector (
   $tags = $lma_collector::params::tags,
   $groups = [],
+  $pre_script = undef,
 ) inherits lma_collector::params {
   include heka::params
   include lma_collector::service
@@ -49,6 +50,7 @@ class lma_collector (
     run_as_root       => $lma_collector::params::run_as_root,
     additional_groups => union($lma_collector::params::groups, $groups),
     hostname          => $::hostname,
+    pre_script        => $pre_script,
   }
 
   file { "${lua_modules_dir}/lma_utils.lua":

deployment_scripts/puppet/modules/lma_collector/templates/wait_for_rabbitmq.erb

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+RABBITMQADMIN="/usr/local/bin/rabbitmqadmin"
+
+# We must wait that rabbitmq is started before running.
+# We can not rely on /var/run/rabbitmq/p_pid because it only means that
+# beam is running.
+# In practice rabbitmqadmin is only present on controller nodes so just skip
+# the waiting loop if we don't find it.
+
+if [ ! -x $RABBITMQADMIN ]; then
+while ! $RABBITMQADMIN -u <%= @rabbitmq_user %> -p <%= @rabbitmq_password %>  show overview >/dev/null 2>&1
+    do
+        echo "$(date +"%Y/%m/%d %H:%M:%S") Waiting for RabbitMQ availability..."
+        sleep <%= @wait_delay %>
+    done
+fi