Add script to stop services and remove the Heka package

This patch creates a new directory for StackLight tools that are scripts
that can be used to perform actions or run diagnostics.
It also adds two scripts. One for stopping StackLight services and
another one to remove the Heka package on all nodes that are online.

Change-Id: I93082ba3c7dfb75ec3ed44f5fa23a35599c4d163
(cherry picked from commit 3b709238a3)
This commit is contained in:
Guillaume Thouvenin 2016-03-25 15:07:55 +01:00
parent f3879c1c6d
commit a129648596
5 changed files with 113 additions and 0 deletions

View File

@ -29,3 +29,9 @@ The LMA dashboards are based on:
* [Grafana](http://grafana.org/) for displaying and querying data in InfluxDB.
To install the dahsboards, see (ui/README.md).
# Tools
Scripts in this directory are:
- diagnostics tools
- helper scripts

11
contrib/tools/README.md Normal file
View File

@ -0,0 +1,11 @@
# Description
Scripts and tools for running diagnostics or actions to manage an environment
deployed with StackLight.
remove_heka_package.sh
: This can be used to remove Heka package on all nodes that are up and ready.
stop_services.sh
: This is used to stop hekad and collectd on all nodes that are up and ready.

24
contrib/tools/common.sh Normal file
View File

@ -0,0 +1,24 @@
#!/bin/bash
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function check_fuel_nodes_file {
if [ ! -f "$1" ]; then
echo "You must first run the following command on the Fuel master node:"
echo " fuel nodes > $1"
exit 1
fi
}

View File

@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FUEL_NODES_FILE=/tmp/nodes
. "$(dirname "$(readlink -f "$0")")"/common.sh
check_fuel_nodes_file "${FUEL_NODES_FILE}"
# Remove Heka due to the issue with heka package versionning
# https://github.com/mozilla-services/heka/issues/1892
echo "** Remove Heka package"
for n in $(grep True $FUEL_NODES_FILE | grep ready |awk -F '|' '{print $5}'); do
echo "$n";
ssh "$n" 'apt-get remove -y heka'
done

44
contrib/tools/stop_services.sh Executable file
View File

@ -0,0 +1,44 @@
#!/bin/bash
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FUEL_NODES_FILE=/tmp/nodes
. "$(dirname "$(readlink -f "$0")")"/common.sh
check_fuel_nodes_file "${FUEL_NODES_FILE}"
# collectd processes could be wedged, stop or kill them
# see https://bugs.launchpad.net/lma-toolchain/+bug/1560946
echo "** Stopping collectd"
for n in $(grep True $FUEL_NODES_FILE | grep ready | awk -F '|' '{print $5}'); do
echo "$n";
ssh "$n" '/etc/init.d/collectd stop; pkill -9 collectd'
done
# Several hekad processes may run on these nodes, stop or kill them
# see https://bugs.launchpad.net/lma-toolchain/+bug/1561109
echo "** Stopping hekad"
for n in $(grep -v controller $FUEL_NODES_FILE | grep True | grep ready | awk -F '|' '{print $5}'); do
echo "$n";
ssh "$n" 'service lma_collector stop; pkill -TERM hekad; sleep 5; pkill -9 hekad;'
done
# Stop hekad on controllers during the upgrade to avoid losing logs and notification
# (because elasticsearch will be stopped and hekad doesn't buffer data with 0.8.0)
echo "** Stopping Heka on controller(s)"
for n in $(grep controller $FUEL_NODES_FILE | grep True | grep ready | awk -F '|' '{print $5}'|tail -n 1); do
echo "$n";
ssh "$n" 'crm resource stop lma_collector'
done