Configure Pacemaker to manage the LMA collector

This change configures Pacemaker to manage the LMA collector service
with proper ordering regarding the local RabbitMQ service.

This also means that I removed the wrapper script that took care of
checking the RabbitMQ availability before launching the hekad process
on the controllers.

Change-Id: I4e747083fb9876f06fde9914b626970e37d0b429
Implements: blueprint lma-aggregator-in-ha-mode

deployment_scripts/puppet/manifests/base.pp

@@ -46,8 +46,12 @@ else {
 
 if hiera('deployment_mode') =~ /^ha_/ and $is_controller {
   $additional_groups = ['haclient']
+  $pacemaker_managed = true
+  $rabbitmq_resource = 'master_p_rabbitmq-server'
 }else{
   $additional_groups = []
+  $pacemaker_managed = false
+  $rabbitmq_resource = undef
 }
 
 $elasticsearch_mode = $lma_collector['elasticsearch_mode']
@@ -69,7 +73,7 @@ case $elasticsearch_mode {
 
 # Notifications are always collected even when event indexation is disabled
 if $is_controller{
-  $pre_script        = '/usr/local/bin/wait_for_rabbitmq'
+  #$pre_script        = '/usr/local/bin/wait_for_rabbitmq'
   # Params used by the script.
   $rabbit            = hiera('rabbit')
   $rabbitmq_port     = hiera('amqp_port', '5673')
@@ -77,14 +81,14 @@ if $is_controller{
   $rabbitmq_password = $rabbit['password']
   $wait_delay        = 30
 
-  file { $pre_script:
-    ensure  => present,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-    content => template('lma_collector/wait_for_rabbitmq.erb'),
-    before  => Class['lma_collector']
-  }
+  #  file { $pre_script:
+  #    ensure  => present,
+  #    owner   => 'root',
+  #    group   => 'root',
+  #    mode    => '0755',
+  #    content => template('lma_collector/wait_for_rabbitmq.erb'),
+  #    before  => Class['lma_collector']
+  #  }
 } else {
   $pre_script = undef
 }
@@ -92,8 +96,10 @@ if $is_controller{
 class { 'lma_collector':
   tags               => merge($tags, $additional_tags),
   groups             => $additional_groups,
-  pre_script         => $pre_script,
+  #  pre_script         => $pre_script,
   aggregator_address => hiera('management_vip'),
+  pacemaker_managed  => $pacemaker_managed,
+  rabbitmq_resource  => $rabbitmq_resource,
 }
 
 if $elasticsearch_mode != 'disabled' {

deployment_scripts/puppet/modules/heka/manifests/init.pp

@@ -75,7 +75,8 @@
 class heka (
   $service_name = $heka::params::service_name,
   $config_dir = $heka::params::config_dir,
-  $run_as_root = $heka::params::run_as_root,
+  $heka_user = $heka::params::user,
+  #$run_as_root = $heka::params::run_as_root,
   $additional_groups = $heka::params::additional_groups,
   $hostname = $heka::params::hostname,
   $maxprocs = $heka::params::maxprocs,
@@ -88,7 +89,7 @@ class heka (
   $internal_statistics = $heka::params::internal_statistics,
 ) inherits heka::params {
 
-  $heka_user     = $heka::params::user
+  $run_as_root   = $heka_user == 'root'
   $hekad_wrapper = "/usr/local/bin/${service_name}_wrapper"
   $base_dir      = "/var/cache/${service_name}"
   $log_file      = "/var/log/${service_name}.log"
@@ -99,14 +100,19 @@ class heka (
   }
 
   if $::osfamily == 'Debian' {
-    # Starting from Heka 0.10.0, the Debian package provides an init script so
-    # stop the service and disable it.
-    service { '_hekad_from_package':
-      ensure  => stopped,
-      name    => 'heka',
-      enable  => false,
+    # Starting from Heka 0.10.0, the Debian package provides a SysV init
+    # script so we need to stop the service and disable it.
+    exec { 'stop_heka_daemon':
+      command => '/etc/init.d/heka stop',
+      onlyif  => '/usr/bin/test -f /etc/init.d/heka',
       require => Package['heka'],
       before  => User['heka'],
+      notify  => Exec['disable_heka_daemon']
+    }
+
+    exec { 'disable_heka_daemon':
+      command     => '/usr/sbin/update-rc.d heka disable',
+      refreshonly => true,
     }
   }
 
@@ -115,13 +121,19 @@ class heka (
     require => Package['heka'],
   }
 
-  user { $heka_user:
-    shell   => '/sbin/nologin',
-    home    => $base_dir,
-    system  => true,
-    groups  => $additional_groups,
-    alias   => 'heka',
-    require => Package['heka'],
+  if ! $run_as_root {
+    # TODO: check if this is really needed
+    user { $heka_user:
+    }
+  } else {
+    user { $heka_user:
+      shell   => '/sbin/nologin',
+      home    => $base_dir,
+      system  => true,
+      groups  => $additional_groups,
+      alias   => 'heka',
+      require => Package['heka'],
+    }
   }
 
   file { $base_dir:

deployment_scripts/puppet/modules/heka/manifests/params.pp

@@ -16,7 +16,6 @@ class heka::params {
   $package_name = 'heka'
   $service_name = 'hekad'
   $user = 'heka'
-  $run_as_root = false
   $additional_groups = []
 
   $hostname = undef

deployment_scripts/puppet/modules/lma_collector/lib/facter/pacemaker_resources.rb

@@ -0,0 +1,34 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+
+# Return the list of resources managed by Pacemaker
+Facter.add('pacemaker_resources') do
+    if Facter::Util::Resolution.which('crm_resource')
+        setcode do
+            # crm_resource -Q -l returns something like this:
+            # vip__public
+            # p_ntp:0
+            # p_ntp:1
+            # p_ntp:2
+            # p_dns:0
+            # ...
+            out = Facter::Util::Resolution.exec('crm_resource -Q -l')
+            # Facter 1.x support only scalar types so we return the list as a
+            # comma-separated string
+            out.split(/\n/).collect{ |x| x.gsub(/:.+$/, '')}.uniq().join(',')
+        end
+    end
+end
+

deployment_scripts/puppet/modules/lma_collector/manifests/init.pp

@@ -33,11 +33,12 @@ class lma_collector (
   $tags = $lma_collector::params::tags,
   $groups = [],
   $pre_script = undef,
+  $pacemaker_managed = $lma_collector::params::pacemaker_managed,
+  $rabbitmq_resource = undef,
   $aggregator_address = undef,
   $aggregator_port = $lma_collector::params::aggregator_port,
 ) inherits lma_collector::params {
   include heka::params
-  include lma_collector::service
 
   validate_hash($tags)
 
@@ -46,10 +47,16 @@ class lma_collector (
   $plugins_dir = $lma_collector::params::plugins_dir
   $lua_modules_dir = $heka::params::lua_modules_dir
 
+  if $lma_collector::params::run_as_root {
+    $heka_user = 'root'
+  } else {
+    $heka_user = $heka::params::user
+  }
+
   class { 'heka':
     service_name        => $service_name,
     config_dir          => $config_dir,
-    run_as_root         => $lma_collector::params::run_as_root,
+    heka_user           => $heka_user,
     additional_groups   => union($lma_collector::params::groups, $groups),
     hostname            => $::hostname,
     pre_script          => $pre_script,
@@ -59,25 +66,98 @@ class lma_collector (
     max_timer_inject    => $lma_collector::params::hekad_max_timer_inject,
   }
 
+  if $pacemaker_managed {
+    validate_string($rabbitmq_resource)
+
+    file { 'ocf-lma_collector':
+      ensure => present,
+      path   => '/usr/lib/ocf/resource.d/fuel/ocf-lma_collector',
+      source => 'puppet:///modules/lma_collector/ocf-lma_collector',
+      mode   => '0755',
+      owner  => 'root',
+      group  => 'root',
+    }
+
+    cs_resource { $service_name:
+      primitive_class => 'ocf',
+      provided_by     => 'fuel',
+      primitive_type  => 'ocf-lma_collector',
+      complex_type    => 'clone',
+      ms_metadata     => {
+        # the resource should start as soon as the dependent resources (eg
+        # RabbitMQ) are running *locally*
+        'interleave' => true,
+      },
+      parameters      => {
+        'config'   => $config_dir,
+        'log_file' => "/var/log/${service_name}.log",
+        'user'     => $heka_user,
+      },
+      operations      => {
+        'monitor' => {
+          'interval' => '10s',
+          'timeout'  => '30s',
+        },
+        'start'   => {
+          'timeout' => '30s',
+        },
+        'stop'    => {
+          'timeout' => '30s',
+        },
+      },
+      require         => [File['ocf-lma_collector'], Class['heka']],
+    }
+
+    cs_rsc_colocation { "${service_name}-with-rabbitmq":
+      ensure     => present,
+      alias      => $service_name,
+      primitives => ["clone_${service_name}", $rabbitmq_resource],
+      score      => 0,
+      require    => Cs_resource[$service_name],
+    }
+
+    cs_rsc_order { "${service_name}-after-rabbitmq":
+      ensure  => present,
+      alias   => $service_name,
+      first   => $rabbitmq_resource,
+      second  => "clone_${service_name}",
+      # Heka cannot start if RabbitMQ isn't ready to accept connections. But
+      # once it is initialized, it can recover from a RabbitMQ outage. This is
+      # why we set score to 0 (interleave) meaning that the collector should
+      # start once RabbitMQ is active but a restart of RabbitMQ
+      # won't trigger a restart of the LMA collector.
+      score   => 0,
+      require => Cs_rsc_colocation[$service_name]
+    }
+
+    class { 'lma_collector::service':
+      provider => 'pacemaker',
+      require  => Cs_rsc_order[$service_name]
+    }
+  } else {
+    # Use the default service class
+    include lma_collector::service
+  }
+
   file { "${lua_modules_dir}/lma_utils.lua":
     ensure  => directory,
     source  => 'puppet:///modules/lma_collector/plugins/common/lma_utils.lua',
     require => File[$lua_modules_dir],
-    notify => Class['lma_collector::service'],
+    notify  => Class['lma_collector::service'],
   }
 
   file { "${lua_modules_dir}/patterns.lua":
     ensure  => directory,
     source  => 'puppet:///modules/lma_collector/plugins/common/patterns.lua',
     require => File[$lua_modules_dir],
-    notify => Class['lma_collector::service'],
+    notify  => Class['lma_collector::service'],
   }
 
   file { "${lua_modules_dir}/extra_fields.lua":
     ensure  => present,
     content => template('lma_collector/extra_fields.lua.erb'),
     require => File[$lua_modules_dir],
-    notify => Class['lma_collector::service'],
+    notify  => Class['lma_collector::service'],
   }
 
   file { $plugins_dir:

deployment_scripts/puppet/modules/lma_collector/manifests/params.pp

@@ -17,6 +17,8 @@ class lma_collector::params {
   $config_dir = "/etc/${service_name}"
   $plugins_dir = "/usr/share/${service_name}"
 
+  $pacemaker_managed = false
+
   # Address and port of the Heka dashboard for health reports.
   $dashboard_address = '127.0.0.1'
   $dashboard_port    = '4352'

deployment_scripts/puppet/modules/lma_collector/manifests/service.pp

@@ -28,6 +28,7 @@ class lma_collector::service (
   $service_enable = true,
   $service_ensure = 'running',
   $service_manage = true,
+  $provider       = undef,
 ) {
   # The base class must be included first because parameter defaults depend on it
   if ! defined(Class['lma_collector::params']) {
@@ -46,11 +47,26 @@ class lma_collector::service (
     }
   }
 
+  if member(split($::pacemaker_resources, ','), $service_name) {
+    $real_provider = 'pacemaker'
+  } else {
+    $real_provider = $provider
+  }
+
   if $service_manage {
-    service { 'lma_collector':
-      ensure => $_service_ensure,
-      name => $service_name,
-      enable => $service_enable,
+    if $real_provider {
+      service { 'lma_collector':
+        ensure   => $_service_ensure,
+        name     => $service_name,
+        enable   => $service_enable,
+        provider => $real_provider,
+      }
+    } else {
+      service { 'lma_collector':
+        ensure => $_service_ensure,
+        name   => $service_name,
+        enable => $service_enable,
+      }
     }
   }
 }