From b5ca79339b9cdfb3e2f09b3fe2e4db551f4dbc57 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Thu, 24 Sep 2015 11:59:35 -0700
Subject: [PATCH] Timeout pre_test, gate, and post_test hooks

Previously devstack-gate only timed out the default gate_hook within the
gate_hook itself. This meant that any other hooks either
pre_test/post_test or non default gate_hooks would not be timed out
properly preventing the copying of logs before Jenkins times out the
build.

We can fix this by wrapping all hook calls with timeouts instead of
expecting the hooks themselves to manage the timeouts. Note that this
requires us to fork and exec a new shell which can be run under timeout
because timeout cannot timeout shell functions.

Change-Id: I1c204d8e91a54df9d39fc6193650cd3e744f1e94
---
 devstack-vm-gate-wrap.sh | 10 +++++-----
 functions.sh             |  9 +++++++++
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/devstack-vm-gate-wrap.sh b/devstack-vm-gate-wrap.sh
index 214865d9..bc9a6b94 100755
--- a/devstack-vm-gate-wrap.sh
+++ b/devstack-vm-gate-wrap.sh
@@ -401,9 +401,9 @@ fi
 if ! function_exists "gate_hook"; then
     # the command we use to run the gate
     function gate_hook {
-        remaining_time
-        timeout -s 9 ${REMAINING_TIME}m $BASE/new/devstack-gate/devstack-vm-gate.sh
+        $BASE/new/devstack-gate/devstack-vm-gate.sh
     }
+    export -f gate_hook
 fi
 
 echo "Triggered by: https://review.openstack.org/$ZUUL_CHANGE patchset $ZUUL_PATCHSET"
@@ -519,11 +519,11 @@ fi
 # Note that hooks should be multihost aware if necessary.
 # devstack-vm-gate-wrap.sh will not automagically run the hooks on each node.
 # Run pre test hook if we have one
-call_hook_if_defined "pre_test_hook"
+timeout_hook call_hook_if_defined "pre_test_hook"
 
 # Run the gate function
 echo "Running gate_hook"
-gate_hook
+timeout_hook "gate_hook"
 GATE_RETVAL=$?
 RETVAL=$GATE_RETVAL
 
@@ -537,7 +537,7 @@ fi
 # Run post test hook if we have one
 if [ $GATE_RETVAL -eq 0 ]; then
     # Run post_test_hook if we have one
-    call_hook_if_defined "post_test_hook"
+    timeout_hook call_hook_if_defined "post_test_hook"
     RETVAL=$?
 fi
 
diff --git a/functions.sh b/functions.sh
index 7f57b584..e74b48ca 100644
--- a/functions.sh
+++ b/functions.sh
@@ -987,3 +987,12 @@ function ovs_gre_bridge {
         fi
     done
 }
+
+# Timeout hook calls implemented as bash functions. Note this
+# forks and execs a new bash in order to use the timeout utility
+# which cannot operate on bash functions directly.
+function timeout_hook {
+    local cmd=$@
+    remaining_time
+    timeout -s 9 ${REMAINING_TIME}m bash -c "source $WORKSPACE/devstack-gate/functions.sh && $cmd"
+}