Add a simple watchdog for logstash-indexer service

The logstash-indexer services sometimes fall out of the elasticsearch
cluster when it is under heavy load and unable to respond to pings.
Logstash doesn't do anything to reconnect :( so restart the service if a
node detects that it has fallen out of the cluster.

Upstream bug submitted at https://logstash.jira.com/browse/LOGSTASH-1951

Change-Id: I2e7767c5fe20cff279366fec2ddadd7710dbb4a9

modules/logstash/files/logstash-watchdog.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# This is a work around for https://logstash.jira.com/browse/LOGSTASH-1951
+# Logstash disconnects from the cluster and will not rejoin under
+# its own power.
+
+ES_ADDRESS=$1
+
+JSON_OUT=$(curl -sf "http://${ES_ADDRESS}:9200/_cluster/nodes/${HOSTNAME}")
+CURL_RET=$?
+RESULT=$(echo $JSON_OUT | jq '.nodes == {}')
+
+if [ "$CURL_RET" == "0" ] && [ "$RESULT" == "true" ] ;
+then
+    stop --quiet logstash-indexer
+    start --quiet logstash-indexer
+fi

modules/logstash/manifests/watchdog.pp

@@ -0,0 +1,41 @@
+# Copyright 2013 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Class to install a simple watchdog for the logstash-indexer service.
+# es_api_node is the address to access the elasticsearch api at (should
+# be a 'host:port' string).
+
+class logstash::watchdog (
+  $es_api_node = 'localhost'
+) {
+  package { 'jq':
+    ensure => present,
+  }
+
+  file { '/usr/local/bin/logstash-watchdog':
+    ensure  => present,
+    source  => 'puppet:///modules/logstash/logstash-watchdog.sh',
+    replace => true,
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0555',
+  }
+
+  cron { 'logstash-watchdog':
+    minute      => '*/10',
+    environment => 'PATH=/bin:/usr/bin:/usr/local/bin',
+    command     => "sleep $((RANDOM\%60)) && /usr/local/bin/logstash-watchdog ${es_api_node}",
+    require     => Service['logstash-indexer']
+  }
+}

modules/openstack_project/manifests/logstash_worker.pp

@@ -39,6 +39,10 @@ class openstack_project::logstash_worker (
     conf_template => 'openstack_project/logstash/indexer.conf.erb',
   }
 
+  class { 'logstash::watchdog':
+    es_api_node => $discover_node,
+  }
+
   include log_processor
   log_processor::worker { 'A':
     config_file => 'puppet:///modules/openstack_project/logstash/jenkins-log-worker.yaml',