[ci] Use stestr for coverage and fail if below 90%

Additionally remove non-voting status of coverage job. 1% toleration is
enabled for allowed_extra_missing lines. If we need to lower the 90%
threshold for coverage in the future, this can be decided then.

Change-Id: I7bc80c34dc6e6e054c61181410d88e7b747310c9

.zuul.yaml

@@ -292,8 +292,6 @@
       - release-notes-jobs-python3
     check:
       jobs:
-        - openstack-tox-cover:
-           voting: false
         - magnum-tempest-plugin-tests-api
         - magnum-container-build
     gate:

tools/cover.sh

@@ -12,11 +12,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-ALLOWED_EXTRA_MISSING=0
+ALLOWED_EXTRA_MISSING_PERCENT=1
 
 show_diff () {
-    head -1 $1
+    result=`diff -U 0 $1 $2 | sed 1,2d`
-    diff -U 0 $1 $2 | sed 1,2d
+    [[ -n "$result" ]] && head -1 $1 || echo "No diff to display"
+    echo "$result"
 }
 
 if ! git diff --exit-code || ! git diff --cached --exit-code
@@ -31,38 +32,49 @@ git checkout HEAD^
 
 base_op_count=`grep "op\." -R magnum/db/sqlalchemy/alembic/versions/ | wc -l`
 baseline_report=$(mktemp -t magnum_coverageXXXXXXX)
-find . -type f -name "*.pyc" -delete && python setup.py testr --coverage --testr-args="$*"
+coverage erase
+find . -type f -name "*.pyc" -delete
+stestr run --no-subunit-trace $*
+coverage combine
 coverage report > $baseline_report
-mv cover cover-master
 cat $baseline_report
-baseline_missing=$(awk 'END { print $3 }' $baseline_report)
+coverage html -d cover-master
+coverage xml -o cover-master/coverage.xml
 
 # Checkout back and save coverage report
 git checkout -
 
 current_op_count=`grep "op\." -R magnum/db/sqlalchemy/alembic/versions/ | wc -l`
 current_report=$(mktemp -t magnum_coverageXXXXXXX)
-find . -type f -name "*.pyc" -delete && python setup.py testr --coverage --testr-args="$*"
+coverage erase
-coverage report > $current_report
+find . -type f -name "*.pyc" -delete
-current_missing=$(awk 'END { print $3 }' $current_report)
+stestr run --no-subunit-trace $*
+coverage combine
+coverage report --fail-under=90 > $current_report
+cat $current_report
+coverage html -d cover
+coverage xml -o cover/coverage.xml
 
 # Show coverage details
-allowed_missing=$((baseline_missing+ALLOWED_EXTRA_MISSING+current_op_count-base_op_count))
+show_diff $baseline_report $current_report > cover/coverage.diff
+cat cover/coverage.diff
+baseline_missing=$(awk 'END { print $3 }' $baseline_report)
+current_missing=$(awk 'END { print $3 }' $current_report)
+allowed_extra_missing=$((baseline_missing*ALLOWED_EXTRA_MISSING_PERCENT/100))
+allowed_missing=$((baseline_missing+allowed_extra_missing+current_op_count-base_op_count))
 
-echo "Allowed to introduce missing lines : ${ALLOWED_EXTRA_MISSING}"
+echo "Allowed to introduce missing lines : ${allowed_extra_missing}"
-echo "Missing lines in master            : ${baseline_missing}"
+echo "Missing lines in baseline          : ${baseline_missing}"
 echo "Missing lines in proposed change   : ${current_missing}"
 
 if [ $allowed_missing -ge $current_missing ]; then
     if [ $baseline_missing -lt $current_missing ]; then
-        show_diff $baseline_report $current_report
         echo "We believe you can test your code with 100% coverage!"
     else
         echo "Thank you! You are awesome! Keep writing unit tests! :)"
     fi
     exit_code=0
 else
-    show_diff $baseline_report $current_report
     echo "Please write more unit tests, we must maintain our test coverage :( "
     exit_code=1
 fi

tox.ini

@@ -114,6 +114,9 @@ deps = -r{toxinidir}/test-requirements.txt
 commands = bandit -r magnum -x tests -n5 -ll
 
 [testenv:cover]
+setenv =
+  {[testenv]setenv}
+  PYTHON=coverage run --source magnum --parallel-mode
 commands = {toxinidir}/tools/cover.sh {posargs}
 
 [testenv:docs]