From 7e4d83e9b2fca211c82b42f304cbec721d34662b Mon Sep 17 00:00:00 2001
From: Nolan Brubaker <nolan.brubaker@rackspace.com>
Date: Tue, 6 Jun 2017 16:04:15 -0400
Subject: [PATCH] Add API availability tests to upgrade

In order to gauge the impact of upgrades on deployment, we will need to
run tests against the various APIs and gather those results with the
gate.

Fetch the API testing script, start it before upgrades, then look for
the logged results.

Change-Id: I6a5e3d42a39610f8772f9524ca2dfa1942955bc9
---
 scripts/gate-check-commit.sh |  9 +++++++++
 scripts/scripts-library.sh   | 30 ++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/scripts/gate-check-commit.sh b/scripts/gate-check-commit.sh
index 4f477ca8ad..5abadcd09d 100755
--- a/scripts/gate-check-commit.sh
+++ b/scripts/gate-check-commit.sh
@@ -189,11 +189,20 @@ if [[ "${ACTION}" == "upgrade" ]]; then
     unset GROUP_VARS_PATH
     unset HOST_VARS_PATH
 
+    # Fetch script to execute API availability tests, then
+    # background them while the upgrade runs.
+    get_bowling_ball_tests
+    start_bowling_ball_tests
+
     # To execute the upgrade script we need to provide
     # an affirmative response to the warning that the
     # upgrade is irreversable.
     echo 'YES' | bash "$(dirname "${0}")/run-upgrade.sh"
 
+    kill_bowling_ball_tests
+    # Wait to let all the processes finish before looking for output.
+    sleep 10
+    print_bowling_ball_results
 fi
 
 exit_success
diff --git a/scripts/scripts-library.sh b/scripts/scripts-library.sh
index 1c970995bf..63eae3f9e9 100755
--- a/scripts/scripts-library.sh
+++ b/scripts/scripts-library.sh
@@ -276,6 +276,36 @@ function get_pip {
   fi
 }
 
+function get_bowling_ball_tests {
+  # Retrieve the latest bowling ball test script in case we don't already have it.
+  if [ -f scripts/rolling_tests.py ]; then
+      return
+  fi
+
+  curl --silent https://raw.githubusercontent.com/openstack/openstack-ansible-ops/master/bowling_ball/rolling_tests.py > scripts/rolling_tests.py
+}
+
+function start_bowling_ball_tests {
+  # The tests will pull Keystone information from the env vars defined in openrc
+  source ~/openrc
+  # Get the list of services to test for from the script, so we're not hard coding.
+  for SERVICE in $(python ./scripts/rolling_tests.py list | cut -d - -f 1); do
+   # Start the scripts in the background and wait between each invocation.
+   # Without the wait, they do not all launch.
+   python ./scripts/rolling_tests.py $SERVICE &
+   sleep 1
+   echo "Started $SERVICE test in background"
+  done
+}
+
+function kill_bowling_ball_tests {
+  pkill -f rolling_tests
+}
+
+function print_bowling_ball_results {
+  grep "failure rate" /var/log/*_rolling.log
+}
+
 ## Signal traps --------------------------------------------------------------
 # Trap all Death Signals and Errors
 trap "exit_fail ${LINENO} $? 'Received STOP Signal'" SIGHUP SIGINT SIGTERM