Port away from ostestr/testr and use stestr

The changes impacts different components:
- unit tests uses stestr directly instead of ostestr;
- the cover target uses stestr+coverage instead of testr;
- sahara-scenario (scenario tests runner) uses stestr
  internally instead of stestr;
- remove the references to testr and ostestr from
  the documentation.
Change-Id: I5eddc4a0f9b245863b4de9056df4c27c876f7123

doc/source/tempest-plugin.rst

@@ -24,39 +24,24 @@ same environment where Tempest is installed.
 
 ..
 
-After that you can run Tempest tests. There you can specify the name of
+After that you can run Tempest tests. You can specify the name of
-test (and even run a single test from directory). One way to run Tempest
+test or a more complex regular expression. While any ``testr``-based
-tests by using ``ostestr`` command, for instance:
+test runner can be used, the official command for executing Tempest
+tests is ``tempest run``.
+
+For example, the following command will run a specific subset of tests:
 
 .. sourcecode:: console
 
-    $ ostestr sahara_tempest_plugin.tests.cli.test_scenario.Scenario.test_plugin_cli
+    $ tempest run --regex '^sahara_tempest_plugin.tests.cli.test_scenario.Scenario.'
 
 ..
 
-The command ``tempest run`` is another way to run tests. See the additional
+The full syntax of ``tempest run`` is described on `the relavant section of
-information about using this `command <https://docs.openstack.org/tempest/latest/run.html>`_.
+the Tempest documentation <https://docs.openstack.org/tempest/latest/run.html>`_.
 
-For example, the following command will run all the Tempest tests.
+Other useful links:
 
-.. sourcecode:: console
-
-    $ tempest run
-
-..
-
-Finally, you can use ``testr`` directly to run the tests. For example,
-the following command will run all the cli-tests:
-
-.. sourcecode:: console
-
-    $ testr run sahara_tempest_plugin.tests.cli
-
-..
-
-Useful links:
-
-* `Running Tempest tests with testr <https://docs.openstack.org/tempest/latest/overview.html#legacy-run-method>`_.
 * `Using Tempest plugins <https://docs.openstack.org/tempest/latest/plugin.html#using-plugins>`_.
 * `Tempest Quickstart <https://docs.openstack.org/tempest/latest/overview.html#quickstart>`_.
 

releasenotes/notes/switch-to-stestr-6de59dc9e46768f6.yaml

@@ -0,0 +1,4 @@
+---
+upgrade:
+  - |
+    sahara-scenario now requires stestr.

requirements.txt

@@ -13,7 +13,6 @@ oslo.serialization>=1.10.0 # Apache-2.0
 oslo.utils>=3.5.0 # Apache-2.0
 oslotest>=1.10.0 # Apache-2.0
 os-client-config>=1.13.1  # Apache-2.0
-os-testr>=0.8.0 # Apache-2.0
 paramiko>=1.16.0 # LGPL
 python-glanceclient>=2.0.0  # Apache-2.0
 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
@@ -22,6 +21,6 @@ python-swiftclient>=2.2.0 # Apache-2.0
 python-neutronclient>=4.2.0 # Apache-2.0
 rfc3986>=0.2.0 # Apache-2.0
 six>=1.9.0 # MIT
+stestr>=1.0.0 # Apache-2.0
 tempest>=16.0.0 # Apache-2.0
-testrepository>=0.0.18 # Apache-2.0/BSD
 testtools>=1.4.0 # MIT

sahara_tests/scenario/stestr.conf

@@ -0,0 +1,3 @@
+[DEFAULT]
+test_path=.
+group_regex=([^\.]+\.)+

sahara_tests/scenario/testr.conf

@@ -1,8 +0,0 @@
-[DEFAULT]
-test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
-             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
-             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
-             ${PYTHON:-python} -m subunit.run discover $DISCOVER_DIRECTORY $LISTOPT $IDOPTION
-test_id_option=--load-list $IDFILE
-test_list_option=--list
-group_regex=([^\.]+\.)+

sahara_tests/scenario/utils.py

@@ -44,7 +44,7 @@ DEFAULT_TEMPLATE_VARS = [os.path.join(TEST_TEMPLATE_DIR,
                                       'edp.yaml.mako')]
 TEST_TEMPLATE_PATH = os.path.join(SCENARIO_RESOURCES_DIR,
                                   'testcase.py.mako')
-DEFAULT_TESTR_CONF = os.path.join(SCENARIO_RESOURCES_DIR, 'testr.conf')
+DEFAULT_STESTR_CONF = os.path.join(SCENARIO_RESOURCES_DIR, 'stestr.conf')
 
 
 def rand_name(name=''):
@@ -56,11 +56,10 @@ def rand_name(name=''):
 
 
 def run_tests(concurrency, test_dir_path):
-    command = ['ostestr']
+    command = ['stestr', 'run']
     if concurrency:
-        command.extend(['--concurrency', '%d' % concurrency])
+        command.extend(['--concurrency=%d' % concurrency])
     new_env = os.environ.copy()
-    new_env['DISCOVER_DIRECTORY'] = '.'
     tester_runner = subprocess.Popen(command, env=new_env, cwd=test_dir_path)
     tester_runner.communicate()
     return tester_runner.returncode
@@ -83,8 +82,9 @@ def create_testcase_file(testcases, credentials, network, report):
     print("The generated test file located at: %s" % test_dir_path)
     fileutils.write_to_tempfile(testcase_data.encode("ASCII"), prefix='test_',
                                 suffix='.py', path=test_dir_path)
-    shutil.copyfile(DEFAULT_TESTR_CONF, os.path.join(test_dir_path,
+    # Copy both files as long as the old runner is supported
-                                                     '.testr.conf'))
+    shutil.copyfile(DEFAULT_STESTR_CONF, os.path.join(test_dir_path,
+                                                      '.stestr.conf'))
 
     return test_dir_path
 

tools/cover.sh

@@ -23,13 +23,20 @@ show_diff () {
 }
 
 package_name=${PACKAGE_NAME:-sahara_tests}
+export PYTHON="coverage run --source ${package_name} --parallel-mode"
+
+run_coverage () {
+    find . -type f -name "*.pyc" -delete && coverage erase && \
+        stestr run "$*" && coverage combine
+}
+
 # Stash uncommitted changes, checkout master and save coverage report
 uncommitted=$(git status --porcelain | grep -v "^??")
 [[ -n $uncommitted ]] && git stash > /dev/null
 git checkout HEAD^
 
 baseline_report=$(mktemp -t sahara-scenario_coverageXXXXXXX)
-find . -type f -name "*.pyc" -delete && python setup.py test --coverage --coverage-package-name=${package_name} --testr-args="$*"
+run_coverage "$*"
 coverage report > $baseline_report
 baseline_missing=$(awk '/^TOTAL/ { print $3 }' $baseline_report)
 
@@ -39,10 +46,13 @@ git checkout -
 
 # Generate and save coverage report
 current_report=$(mktemp -t sahara-scenario_coverageXXXXXXX)
-find . -type f -name "*.pyc" -delete && python setup.py test --coverage --coverage-package-name=${package_name} --testr-args="$*"
+run_coverage "$*"
 coverage report > $current_report
 current_missing=$(awk '/^TOTAL/ { print $3 }' $current_report)
 
+coverage html -d cover
+coverage xml -o cover/coverage.xml
+
 # Show coverage details
 allowed_missing=$((baseline_missing+ALLOWED_EXTRA_MISSING))
 
@@ -70,4 +80,5 @@ else
 fi
 
 rm $baseline_report $current_report
+
 exit $exit_code

tox.ini

@@ -8,11 +8,10 @@ usedevelop = True
 install_command = pip install -U {opts} {packages}
 setenv =
     VIRTUAL_ENV={envdir}
-    DISCOVER_DIRECTORY=sahara_tests/unit/scenario
 deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/test-requirements.txt
-commands = ostestr {posargs}
+commands = stestr run {posargs}
 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY OS_*
 
 [testenv:venv]
@@ -22,7 +21,6 @@ passenv = OS_*
 [testenv:cover]
 basepython = python3
 setenv =
-    DISCOVER_DIRECTORY=sahara_tests
     PACKAGE_NAME=sahara_tests
 commands = {toxinidir}/tools/cover.sh {posargs}