From 899fb6e368c746cb7e27751b2517ccae3ce05b40 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 4 Apr 2018 13:19:22 +1000 Subject: [PATCH] Add a backoff for failed builds If a build has a systematic failure, we currently just let nodepool run, looping builds until we hit the overall job timeout (1.5 hours). This adds a count of output log files; if we see three failed builds, then assume the problem won't get better and fail early. Change-Id: Id7e163b4937dd57cc8afbf72ed795f73b46a05b1 --- tools/check_devstack_plugin.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/check_devstack_plugin.sh b/tools/check_devstack_plugin.sh index e48a636d8..8334bb8b3 100755 --- a/tools/check_devstack_plugin.sh +++ b/tools/check_devstack_plugin.sh @@ -47,12 +47,19 @@ function sshintonode { } function waitforimage { - name=$1 - state='ready' + local name=$1 + local state='ready' + local builds while ! $NODEPOOL image-list | grep $name | grep $state; do $NODEPOOL image-list > ${LOGDIR}/nodepool-image-list.txt $NODEPOOL list > ${LOGDIR}/nodepool-list.txt + + builds=$(ls -l /var/log/nodepool/builds/ | grep $name | wc -l) + if [[ ${builds} -ge 4 ]]; then + echo "*** Build of $name failed at least 3 times, aborting" + exit 1 + fi sleep 10 done }