Add VIPs cluster location metrics

The pacemaker command 'crm  resource --locate --resource <rsrc>' is used to
collect 'vip__public' and 'vip__management' locations.

The command is executed by a ProcessInput Heka plugin.

Change-Id: I3a667931a58809b84d667592a7e618b906eddc56

deployment_scripts/puppet/manifests/base.pp

@@ -19,8 +19,15 @@ else {
   $additional_tags = {}
 }
 
+if hiera('deployment_mode') =~ /^ha_/ and hiera('role') =~ /controller/{
+  $additional_groups = ['haclient']
+}else{
+  $additional_groups = []
+}
+
 class { 'lma_collector':
-  tags => merge($tags, $additional_tags)
+  tags   => merge($tags, $additional_tags),
+  groups => $additional_groups,
 }
 
 class { 'lma_collector::logs::system':

deployment_scripts/puppet/manifests/controller.pp

@@ -101,4 +101,9 @@ if $lma_collector['influxdb_mode'] != 'disabled' {
   class { 'lma_collector::metrics::service_heartbeat':
     services => ['mysql', 'rabbitmq', 'haproxy', 'memcached', 'apache']
   }
+
+  # Enable pacemaker resource location metrics
+  if $ha_deployment {
+    class { 'lma_collector::metrics::pacemaker_resources': }
+  }
 }

deployment_scripts/puppet/modules/heka/manifests/input/process.pp

@@ -0,0 +1,21 @@
+define heka::input::process (
+  $config_dir,
+  $decoder,
+  $commands,
+  $splitter = false,
+  $ticker_interval = '60',
+  $stdout = true,
+  $stderr = false,
+  $ensure = present,
+) {
+
+  include heka::params
+
+  file { "${config_dir}/process-${title}.toml":
+    ensure  => $ensure,
+    content => template('heka/input/process.toml.erb'),
+    mode    => '0600',
+    owner   => $heka::params::user,
+    group   => $heka::params::user,
+  }
+}

deployment_scripts/puppet/modules/heka/templates/input/process.toml.erb

@@ -0,0 +1,28 @@
+[<%= @title %>_process]
+type="ProcessInput"
+ticker_interval = <%= @ticker_interval %>
+decoder = "<%= @decoder %>_decoder"
+<% if @stdout -%>
+stdout = true
+<% else %>
+stdout = false
+<% end %>
+<% if @stderr -%>
+stderr = true
+<% else %>
+stderr = false
+<% end %>
+
+<% if @splitter -%>
+splitter = "<%= @splitter %>_splitter"
+<% else -%>
+splitter = "NullSplitter"
+<% end -%>
+
+<% @commands.each_with_index do |command, i| -%>
+ <% command.each do |cmd, args| -%>
+  [<%= @title %>_process.command.<%= i %>]
+   bin = "<%= cmd %>"
+   args = [ <%= args.collect{ |x| '"%s"' % x }.join(", ") %> ]
+ <%end%>
+<% end %>

deployment_scripts/puppet/modules/lma_collector/files/pacemaker/locate_resources.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Copyright 2015 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# produce the following output for all VIP resources:
+# <resource-name> <node> <active>
+#
+# where:
+#  <resource-name>: name of the resource (ie vip__public)
+#  <node>: the hostname of the node where the resource is located (ie node-1)
+#  <active>: either '0' or '1' if <node> matches the local hostname
+
+host=$(hostname -s)
+
+for rsr in vip__public vip__management vip__public_vrouter vip__management_vrouter; do
+  node=$(/usr/sbin/crm_resource --locate --quiet --resource  $rsr 2>/dev/null)
+  if [ $? -eq 0 ]; then
+    if [[ x"$host" = x"$node" ]]; then
+        iam=1
+    else
+        iam=0
+    fi
+    echo $rsr $node $iam
+  fi
+done
+exit 0

deployment_scripts/puppet/modules/lma_collector/files/plugins/decoders/pacemaker_resources.lua

@@ -0,0 +1,53 @@
+-- Copyright 2015 Mirantis, Inc.
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+require 'string'
+local l = require 'lpeg'
+local utils = require 'lma_utils'
+l.locale(l)
+
+local msg = {
+    Timestamp = nil,
+    Type = "metric",
+    Payload = nil,
+    Severity = 6, -- INFO
+    Fields = nil
+}
+
+local word = (l.R("az", "AZ", "09") + l.P"." + l.P"_" + l.P"-")^1
+local grammar = l.Ct(l.Cg(word, 'resource') * " " * l.Cg(word, 'node') * " " * l.Cg(l.xdigit, 'active'))
+
+function process_message ()
+    local data = read_message("Payload")
+    local m = grammar:match(data)
+    if not m then
+        return -1
+    end
+    msg.Timestamp = read_message("Timestamp")
+    msg.Payload = data
+    msg.Fields = {}
+    msg.Fields.source = 'pacemaker'
+    msg.Fields.type = utils.metric_type['GAUGE']
+    msg.Fields.hostname = read_message('Hostname')
+    utils.inject_tags(msg)
+
+    msg.Fields.name = string.format('pacemaker.resource.%s.active_node', m.resource)
+    msg.Fields.value = m.node
+    inject_message(msg)
+
+    msg.Fields.name= string.format('pacemaker.resource.%s.active', m.resource)
+    msg.Fields.value = tonumber(m.active)
+    inject_message(msg)
+
+    return 0
+end

deployment_scripts/puppet/modules/lma_collector/manifests/init.pp

@@ -17,6 +17,7 @@
 #
 class lma_collector (
   $tags = $lma_collector::params::tags,
+  $groups = [],
 ) inherits lma_collector::params {
   include heka::params
   include lma_collector::service
@@ -32,7 +33,7 @@ class lma_collector (
     service_name      => $service_name,
     config_dir        => $config_dir,
     run_as_root       => $lma_collector::params::run_as_root,
-    additional_groups => $lma_collector::params::groups,
+    additional_groups => union($lma_collector::params::groups, $groups),
     hostname          => $::hostname,
   }
 

deployment_scripts/puppet/modules/lma_collector/manifests/metrics/pacemaker_resources.pp

@@ -0,0 +1,36 @@
+class lma_collector::metrics::pacemaker_resources (
+  $interval = $lma_collector::params::pacemaker_resources_interval,
+) inherits lma_collector::params {
+
+  include heka::params
+
+  file { $lma_collector::params::pacemaker_resources_script:
+    ensure  => present,
+    source  => 'puppet:///modules/lma_collector/pacemaker/locate_resources.sh',
+    mode    => '0750',
+    owner   => $heka::params::user,
+    group   => $heka::params::user,
+  }
+
+  heka::splitter::token { 'pacemaker_resource':
+    config_dir => $lma_collector::params::config_dir,
+    delimiter  => '\n',
+  }
+
+  $pacemaker_resource_cmd = {"${lma_collector::params::pacemaker_resources_script}" => []}
+
+  heka::input::process { 'pacemaker_resource':
+    config_dir        => $lma_collector::params::config_dir,
+    commands          => [$pacemaker_resource_cmd],
+    decoder           => 'pacemaker_resource',
+    splitter          => 'pacemaker_resource',
+    ticker_interval   => $interval,
+    notify            => Class['lma_collector::service'],
+  }
+
+  heka::decoder::sandbox { 'pacemaker_resource':
+    config_dir  => $lma_collector::params::config_dir,
+    filename    => "${lma_collector::params::plugins_dir}/decoders/pacemaker_resources.lua",
+    notify      => Class['lma_collector::service'],
+  }
+}

deployment_scripts/puppet/modules/lma_collector/manifests/params.pp

@@ -71,4 +71,7 @@ class lma_collector::params {
   $influxdb_user = 'lma'
   $influxdb_password = 'lmapass'
   $influxdb_timeout = 5
+
+  $pacemaker_resources_script = '/usr/local/bin/pacemaker_locate_resources.sh'
+  $pacemaker_resources_interval = '60'
 }

doc/source/conf.py

@@ -76,6 +76,7 @@ exclude_patterns = [
     'metrics/openstack.rst',
     'metrics/rabbitmq.rst',
     'metrics/system.rst',
+    'metrics/pacemaker.rst',
 ]
 
 # The reST default role (used for this markup: `text`) to use for all

doc/source/metrics.rst

@@ -109,3 +109,8 @@ Ceph
 ----
 
 .. include:: metrics/ceph.rst
+
+Pacemaker
+---------
+
+.. include:: metrics/pacemaker.rst

doc/source/metrics/pacemaker.rst

@@ -0,0 +1,12 @@
+.. _pacemaker-metrics:
+
+Resource location
+^^^^^^^^^^^^^^^^^
+
+* ``pacemaker.resource.<resource-name>.active``,  ``1`` when the resource is
+  located on the host reporting the metric, ``0`` otherwise.
+* ``pacemaker.resource.<resource-name>.active_node``, the hostname where
+  the resource is located.
+
+``<resource-name>`` is one of 'vip__public', 'vip__management',
+  'vip__public_vrouter' or 'vip__management_vrouter'.