Add coverage test script

Story: 2001595
Task: 6545
Change-Id: Ib97c3b281377c30008abb9bd2d1fe78c515c91ef

tools/cover.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Copyright 2018 Catalyst IT Ltd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+ALLOWED_EXTRA_MISSING=5
+TESTR_ARGS="$*"
+
+show_diff () {
+    head -1 $1
+    diff -U 0 $1 $2 | sed 1,2d
+}
+
+# Stash uncommitted changes, checkout master and save coverage report
+uncommitted=$(git status --porcelain | grep -v "^??")
+[[ -n $uncommitted ]] && git stash > /dev/null
+git checkout HEAD^
+
+baseline_report=$(mktemp -t qinling_coverageXXXXXXX)
+find . -type f -name "*.py[c|o]" -delete && stestr run "$TESTR_ARGS" && coverage combine && coverage html -d cover
+coverage report --ignore-errors > $baseline_report
+baseline_missing=$(awk 'END { print $3 }' $baseline_report)
+
+# Checkout back and unstash uncommitted changes (if any)
+git checkout -
+[[ -n $uncommitted ]] && git stash pop > /dev/null
+
+# Generate and save coverage report
+current_report=$(mktemp -t qingling_coverageXXXXXXX)
+find . -type f -name "*.py[c|o]" -delete && stestr run "$TESTR_ARGS" && coverage combine && coverage html -d cover
+coverage report --ignore-errors > $current_report
+current_missing=$(awk 'END { print $3 }' $current_report)
+
+# Show coverage details
+allowed_missing=$((baseline_missing+ALLOWED_EXTRA_MISSING))
+
+echo "Allowed to introduce missing lines : ${ALLOWED_EXTRA_MISSING}"
+echo "Missing lines in master            : ${baseline_missing}"
+echo "Missing lines in proposed change   : ${current_missing}"
+
+if [ $allowed_missing -gt $current_missing ]; then
+    if [ $baseline_missing -lt $current_missing ]; then
+        show_diff $baseline_report $current_report
+        echo "I believe you can cover all your code with 100% coverage!"
+    else
+        echo "Thank you! You are awesome! Keep writing unit tests! :)"
+    fi
+    exit_code=0
+else
+    show_diff $baseline_report $current_report
+    echo "Please write more unit tests, we should keep our test coverage :( "
+    exit_code=1
+fi
+
+rm -f $baseline_report $current_report
+exit $exit_code

tox.ini

@@ -35,10 +35,7 @@ setenv =
     {[testenv]setenv}
     PYTHON=coverage run --source qinling --parallel-mode
 commands =
-    stestr run {posargs}
+    {toxinidir}/tools/cover.sh {posargs}
-    coverage combine
-    coverage html -d cover
-    coverage xml -o cover/coverage.xml
 
 [testenv:docs]
 whitelist_externals = rm