Grab additional data about IO to profile service timeouts

We are hitting timeouts in IO bound processes as swift but we
don't have much information about what processes are causing
this IO peaks.

This patch adds iotop and iostat information to the logs in
addition to dstat output.

Change-Id: I4c50d4405a923eb2652fc39f6cea9889597ae44e

copy_logs.sh

@@ -114,6 +114,16 @@ if [ -f /var/log/dstat.log ] ; then
     sudo cp /var/log/dstat.log $LOG_DIR/
 fi
 
+# iostat logs
+if [ -f /var/log/iostat.log ] ; then
+    sudo cp /var/log/iostat.log $LOG_DIR/
+fi
+
+# iotop logs
+if [ -f /var/log/iotop.log ] ; then
+    sudo cp /var/log/iotop.log $LOG_DIR/
+fi
+
 # libvirt
 if [ -d /var/log/libvirt ] ; then
     sudo cp -r /var/log/libvirt $LOG_DIR/

run_tests.sh

@@ -140,9 +140,9 @@ if [ "${MANAGE_HIERA}" = true ]; then
 fi
 
 if uses_debs; then
-    $SUDO apt-get install -y dstat ebtables
+    $SUDO apt-get install -y dstat ebtables iotop sysstat
 elif is_fedora; then
-    $SUDO yum install -y dstat setools setroubleshoot audit
+    $SUDO yum install -y dstat setools setroubleshoot audit iotop sysstat
     $SUDO service auditd start
     # SElinux in permissive mode so later we can catch alerts
     $SUDO selinuxenabled && $SUDO setenforce 0
@@ -154,6 +154,16 @@ if type "dstat" 2>/dev/null; then
     $SUDO dstat -tcmndrylpg --top-cpu-adv --top-io-adv --nocolor | $SUDO tee --append /var/log/dstat.log > /dev/null &
 fi
 
+if type "iostat" 2>/dev/null; then
+    print_header 'Start iostat'
+    $SUDO iostat -x -k -d -t 4 | $SUDO tee --append /var/log/iostat.log > /dev/null &
+fi
+
+if [ -f "/usr/sbin/iotop" ]; then
+    print_header 'Start iotop'
+    $SUDO /usr/sbin/iotop --kilobytes --only --batch --time --delay=2 --processes --quiet | $SUDO tee --append /var/log/iotop.log > /dev/null &
+fi
+
 if [ "${MANAGE_PUPPET_MODULES}" = true ]; then
     $SUDO ./install_modules.sh
 fi