add gating up/down script for devstack

This adds the test infrastructure for testing that unstack.sh and
clean.sh do the right thing, and actually stop what's expected. This
is designed to be used in upstream testing to make unstack and clean a
bit more certain.

It includes numerous fixes to make these pass in an errexit
environment with the gate config. The scripts still don't run under
errexit because we don't assume we've handled all possible cleanup safely.

Change-Id: I774dfb2cc934367eef2bb7ea5123197f6da7565b

clean.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # **clean.sh**
 
@@ -83,7 +83,10 @@ if [[ -d $TOP_DIR/extras.d ]]; then
 fi
 
 # Clean projects
-cleanup_cinder
+
+# BUG: cinder tgt doesn't exit cleanly if it's not running.
+cleanup_cinder || /bin/true
+
 cleanup_glance
 cleanup_keystone
 cleanup_nova

gate/updown.sh

@@ -0,0 +1,24 @@
+#!/bin/bash -xe
+#
+# An up / down test for gate functional testing
+#
+# Note: this is expected to start running as jenkins
+
+# Step 1: give back sudoers permissions to devstack
+TEMPFILE=`mktemp`
+echo "stack ALL=(root) NOPASSWD:ALL" >$TEMPFILE
+chmod 0440 $TEMPFILE
+sudo chown root:root $TEMPFILE
+sudo mv $TEMPFILE /etc/sudoers.d/51_stack_sh
+
+# TODO: do something to start a guest to create crud that should
+# disappear
+
+# Step 2: unstack
+echo "Running unstack.sh"
+sudo -H -u stack stdbuf -oL -eL bash -ex ./unstack.sh
+
+# Step 3: clean
+echo "Running clean.sh"
+sudo -H -u stack stdbuf -oL -eL bash -ex ./clean.sh
+

lib/databases/mysql

@@ -28,17 +28,14 @@ function cleanup_database_mysql {
     stop_service $MYSQL
     if is_ubuntu; then
         # Get ruthless with mysql
-        stop_service $MYSQL
         apt_get purge -y mysql* mariadb*
         sudo rm -rf /var/lib/mysql
         sudo rm -rf /etc/mysql
         return
     elif is_fedora; then
-        stop_service mariadb
         uninstall_package mariadb-server
         sudo rm -rf /var/lib/mysql
     elif is_suse; then
-        stop_service mysql
         uninstall_package mysql-community-server
         sudo rm -rf /var/lib/mysql
     else

lib/dstat

@@ -40,7 +40,10 @@ function start_dstat {
 
 # stop_dstat() stop dstat process
 function stop_dstat {
-    screen_stop dstat
+    # dstat runs as a console, not as a service, and isn't trackable
+    # via the normal mechanisms for devstack. So lets just do a
+    # killall and move on.
+    killall dstat || /bin/true
 }
 
 # Restore xtrace

lib/rpc_backend

@@ -74,7 +74,8 @@ function cleanup_rpc_backend {
     if is_service_enabled rabbit; then
         # Obliterate rabbitmq-server
         uninstall_package rabbitmq-server
-        sudo killall epmd || sudo killall -9 epmd
+        # in case it's not actually running, /bin/true at the end
+        sudo killall epmd || sudo killall -9 epmd || /bin/true
         if is_ubuntu; then
             # And the Erlang runtime too
             apt_get purge -y erlang*

lib/sahara

@@ -184,7 +184,7 @@ function start_sahara {
 # stop_sahara() - Stop running processes
 function stop_sahara {
     # Kill the Sahara screen windows
-    screen -S $SCREEN_NAME -p sahara -X kill
+    stop_process sahara
 }
 
 

unstack.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # **unstack.sh**
 
@@ -136,10 +136,13 @@ fi
 
 SCSI_PERSIST_DIR=$CINDER_STATE_PATH/volumes/*
 
+# BUG: tgt likes to exit 1 on service stop if everything isn't
+# perfect, we should clean up cinder stop paths.
+
 # Get the iSCSI volumes
 if is_service_enabled cinder; then
-    stop_cinder
-    cleanup_cinder
+    stop_cinder || /bin/true
+    cleanup_cinder || /bin/true
 fi
 
 if [[ -n "$UNSTACK_ALL" ]]; then
@@ -179,4 +182,5 @@ if [[ -n "$SCREEN" ]]; then
     fi
 fi
 
-clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME
+# BUG: maybe it doesn't exist? We should isolate this further down.
+clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME || /bin/true