Add script to stop services and remove the Heka package

This patch creates a new directory for StackLight tools that are scripts
that can be used to perform actions or run diagnostics.
It also adds two scripts. One for stopping StackLight services and
another one to remove the Heka package on all nodes that are online.

Change-Id: I93082ba3c7dfb75ec3ed44f5fa23a35599c4d163

contrib/tools/README.md

@@ -0,0 +1,11 @@
+# Description
+
+Scripts and tools for running diagnostics or actions to manage an environment
+deployed with StackLight.
+
+remove_heka_package.sh
+: This can be used to remove Heka package on all nodes that are up and ready.
+
+
+stop_services.sh
+: This is used to stop hekad and collectd on all nodes that are up and ready.

contrib/tools/common.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function check_fuel_nodes_file {
+    if [ ! -f "$1" ]; then
+      echo "You must first run the following command on the Fuel master node:"
+      echo "  fuel nodes > $1"
+      exit 1
+    fi
+}
+
+

contrib/tools/remove_heka_package.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FUEL_NODES_FILE=/tmp/nodes
+
+. "$(dirname "$(readlink -f "$0")")"/common.sh
+
+check_fuel_nodes_file "${FUEL_NODES_FILE}"
+
+# Remove Heka due to the issue with heka package versionning
+# https://github.com/mozilla-services/heka/issues/1892
+echo "** Remove Heka package"
+for n in $(grep True $FUEL_NODES_FILE | grep ready |awk -F '|' '{print $5}'); do
+    echo "$n";
+    ssh "$n" 'apt-get remove -y heka'
+done

contrib/tools/stop_services.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright 2016 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FUEL_NODES_FILE=/tmp/nodes
+
+. "$(dirname "$(readlink -f "$0")")"/common.sh
+
+check_fuel_nodes_file "${FUEL_NODES_FILE}"
+
+# collectd processes could be wedged, stop or kill them
+# see https://bugs.launchpad.net/lma-toolchain/+bug/1560946
+echo "** Stopping collectd"
+for n in $(grep True $FUEL_NODES_FILE | grep ready | awk -F '|' '{print $5}'); do
+    echo "$n";
+    ssh "$n" '/etc/init.d/collectd  stop; pkill -9 collectd'
+done
+
+# Several hekad processes may run on these nodes, stop or kill them
+# see https://bugs.launchpad.net/lma-toolchain/+bug/1561109
+echo "** Stopping hekad"
+for n in $(grep -v controller $FUEL_NODES_FILE | grep True | grep ready | awk -F '|' '{print $5}'); do
+    echo "$n";
+    ssh "$n" 'service lma_collector stop; pkill -TERM hekad; sleep 5; pkill -9 hekad;'
+done
+
+# Stop hekad on controllers during the upgrade to avoid losing logs and notification
+# (because elasticsearch will be stopped and hekad doesn't buffer data with 0.8.0)
+echo "** Stopping Heka on controller(s)"
+for n in $(grep controller $FUEL_NODES_FILE | grep True | grep ready | awk -F '|' '{print $5}'|tail -n 1); do
+    echo "$n";
+    ssh "$n" 'crm resource stop lma_collector'
+done