Timeout pre_test, gate, and post_test hooks

Previously devstack-gate only timed out the default gate_hook within the
gate_hook itself. This meant that any other hooks either
pre_test/post_test or non default gate_hooks would not be timed out
properly preventing the copying of logs before Jenkins times out the
build.

We can fix this by wrapping all hook calls with timeouts instead of
expecting the hooks themselves to manage the timeouts. Note that this
requires us to fork and exec a new shell which can be run under timeout
because timeout cannot timeout shell functions.

Change-Id: I1c204d8e91a54df9d39fc6193650cd3e744f1e94

devstack-vm-gate-wrap.sh

@@ -401,9 +401,9 @@ fi
 if ! function_exists "gate_hook"; then
     # the command we use to run the gate
     function gate_hook {
-        remaining_time
-        timeout -s 9 ${REMAINING_TIME}m $BASE/new/devstack-gate/devstack-vm-gate.sh
+        $BASE/new/devstack-gate/devstack-vm-gate.sh
     }
+    export -f gate_hook
 fi
 
 echo "Triggered by: https://review.openstack.org/$ZUUL_CHANGE patchset $ZUUL_PATCHSET"
@@ -519,11 +519,11 @@ fi
 # Note that hooks should be multihost aware if necessary.
 # devstack-vm-gate-wrap.sh will not automagically run the hooks on each node.
 # Run pre test hook if we have one
-call_hook_if_defined "pre_test_hook"
+timeout_hook call_hook_if_defined "pre_test_hook"
 
 # Run the gate function
 echo "Running gate_hook"
-gate_hook
+timeout_hook "gate_hook"
 GATE_RETVAL=$?
 RETVAL=$GATE_RETVAL
 
@@ -537,7 +537,7 @@ fi
 # Run post test hook if we have one
 if [ $GATE_RETVAL -eq 0 ]; then
     # Run post_test_hook if we have one
-    call_hook_if_defined "post_test_hook"
+    timeout_hook call_hook_if_defined "post_test_hook"
     RETVAL=$?
 fi
 

functions.sh

@@ -987,3 +987,12 @@ function ovs_gre_bridge {
         fi
     done
 }
+
+# Timeout hook calls implemented as bash functions. Note this
+# forks and execs a new bash in order to use the timeout utility
+# which cannot operate on bash functions directly.
+function timeout_hook {
+    local cmd=$@
+    remaining_time
+    timeout -s 9 ${REMAINING_TIME}m bash -c "source $WORKSPACE/devstack-gate/functions.sh && $cmd"
+}