turn back on log enforcement

This turns back on the log enforcement that certain logs need to
be clean. This will fail if a new log gets added that has errors.

The logic was cleaned up a bit to rely on -o errexit. While we
should already be at -o errexit at that part of the script, an
added call is made to ensure if anyone changes the top half they
don't change the behavior of this part.

We also create a specific out for 3rd party CI, or other tests
that don't want this enforcing, which is to set
DEVSTACK_GATE_CLEAN_LOGS=0 in the environment.

Change-Id: I50944ad64bec948504a0124ea423a0f96f8a8361

devstack-vm-gate-wrap.sh

@@ -240,6 +240,10 @@ export DEVSTACK_GATE_REQS_INTEGRATION=${DEVSTACK_GATE_REQS_INTEGRATION:-0}
 # nonexistant location to disable mirror selection
 export DEVSTACK_GATE_SELECT_MIRROR=${DEVSTACK_GATE_SELECT_MIRROR:-/usr/local/jenkins/slave_scripts/select-mirror.sh}
 
+# Set to 0 to disable clean logs enforcement (3rd party CI might want to do this
+# until they get their driver cleaned up)
+export DEVSTACK_GATE_CLEAN_LOGS=${DEVSTACK_GATE_CLEAN_LOGS:-1}
+
 # Set this to the time in minutes that the gate test should be allowed
 # to run before being aborted (default 60).
 export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-60}

devstack-vm-gate.sh

@@ -322,62 +322,52 @@ if [ "$DEVSTACK_GATE_TEMPEST" -eq "1" ]; then
         sudo chmod -R o+rx /opt/stack/new/devstack/files
     fi
 
-    # let us control if we die or not
-    set +o errexit
+    # From here until the end we rely on the fact that all the code fails if
+    # something is wrong, to enforce exit on bad test results.
+    set -o errexit
 
     cd $BASE/new/tempest
     if [[ "$DEVSTACK_GATE_TEMPEST_REGEX" != "" ]] ; then
         echo "Running tempest with a custom regex filter"
         sudo -H -u tempest tox -eall -- --concurrency=$TEMPEST_CONCURRENCY $DEVSTACK_GATE_TEMPEST_REGEX
-        res=$?
     elif [[ "$DEVSTACK_GATE_TEMPEST_ALL" -eq "1" ]]; then
         echo "Running tempest all test suite"
         sudo -H -u tempest tox -eall -- --concurrency=$TEMPEST_CONCURRENCY
-        res=$?
     elif [[ "$DEVSTACK_GATE_TEMPEST_DISABLE_TENANT_ISOLATION" -eq "1" ]]; then
         echo "Running tempest full test suite serially"
         sudo -H -u tempest tox -efull-serial
-        res=$?
     elif [[ "$DEVSTACK_GATE_TEMPEST_FULL" -eq "1" ]]; then
         echo "Running tempest full test suite"
         sudo -H -u tempest tox -efull -- --concurrency=$TEMPEST_CONCURRENCY
-        res=$?
     elif [[ "$DEVSTACK_GATE_TEMPEST_TESTR_FULL" -eq "1" ]]; then
         echo "Running tempest full test suite with testr"
         sudo -H -u tempest tox -etestr-full -- --concurrency=$TEMPEST_CONCURRENCY
-        res=$?
     elif [[ "$DEVSTACK_GATE_TEMPEST_STRESS" -eq "1" ]] ; then
         echo "Running stress tests"
         sudo -H -u tempest tox -estress
-        res=$?
     elif [[ "$DEVSTACK_GATE_TEMPEST_HEAT_SLOW" -eq "1" ]] ; then
         echo "Running slow heat tests"
         sudo -H -u tempest tox -eheat-slow -- --concurrency=$TEMPEST_CONCURRENCY
-        res=$?
     elif [[ "$DEVSTACK_GATE_TEMPEST_LARGE_OPS" -ge "1" ]] ; then
         echo "Running large ops tests"
         sudo -H -u tempest tox -elarge-ops -- --concurrency=$TEMPEST_CONCURRENCY
-        res=$?
     elif [[ "$DEVSTACK_GATE_SMOKE_SERIAL" -eq "1" ]] ; then
         echo "Running tempest smoke tests"
         sudo -H -u tempest tox -esmoke-serial
-        res=$?
     else
         echo "Running tempest smoke tests"
         sudo -H -u tempest tox -esmoke -- --concurrency=$TEMPEST_CONCURRENCY
-        res=$?
     fi
 
-    if [[ "$DEVSTACK_GATE_TEMPEST_STRESS" -ne "1" ]] ; then
-        tools/check_logs.py -d $BASE/new/screen-logs
-        res2=$?
+    if [[ "$DEVSTACK_GATE_CLEAN_LOGS" -eq "0" ]] ; then
+        # if we don't want to enforce clean logs, just turn off
+        # errexit on this final command
+        set +o errexit
     fi
-    # TODO(sdague): post icehouse-2 we can talk about turning
-    # this back on, but right now it is violating the do no harm
-    # principle.
-    # [[ $res -eq 0 && $res2 -eq 0 ]]
-    # exit $?
-    exit $res
+
+    echo "Running log checker"
+    tools/check_logs.py -d $BASE/new/screen-logs
+
 
 else
     # Jenkins expects at least one nosetests file.  If we're not running