Add simple wrapper for pytest

It helps to:
 - publish report at upstream CI(mask report as testr-report)
 - launch single tests. Returns ability to launch single test or group of
   tests like:
      tox -e py27 -- tests.unit.test_osclients.CachedTestCase.test_cached

Change-Id: I2463360bc2f0559bcdd830ec6f4ba26cc51c80c3

doc/source/contribute.rst

@@ -142,7 +142,7 @@ To run a single unit test e.g. test_deployment
   $ tox -e <name> -- <test_name>
 
   #NOTE: <name> is one of py34, py27 or pep8
-  #      <test_name> is the unit test case name
+  #      <test_name> is the unit test case name, e.g tests.unit.test_osclients
 
 To debug issues on the unit test:
 

tests/README.rst

@@ -25,28 +25,32 @@ To run unit tests locally::
   $ pip install tox
   $ tox
 
-To run py26, py27 or pep8 only::
+To run py27, py34, py35 or pep8 only::
 
   $ tox -e <name>
 
-  #NOTE: <name> is one of py26, py27 or pep8
+  # NOTE: <name> is one of py27, py34, py35 or pep8
 
-To run py26, py27 against mysql or psql
+To run py27/py34/py35 against mysql or psql
 
   $ export RALLY_UNITTEST_DB_URL="mysql://user:secret@localhost/rally"
   $ tox -epy27
 
+To run specific test of py27/py34/py35::
+
+  $ tox -e py27 -- tests.unit.test_osclients
+
 To get test coverage::
 
   $ tox -e cover
 
-  #NOTE: Results will be in /cover/index.html
+  # NOTE: Results will be in ./cover/index.html
 
 To generate docs::
 
   $ tox -e docs
 
-  #NOTE: Documentation will be in doc/source/_build/html/index.html
+  # NOTE: Documentation will be in doc/source/_build/html/index.html
 
 Functional tests
 ----------------
@@ -62,7 +66,7 @@ To run functional tests locally::
   $ rally deployment create --fromenv --name testing
   $ tox -e cli
 
-  #NOTE: openrc file with OpenStack admin credentials
+  # NOTE: openrc file with OpenStack admin credentials
 
 Output of every Rally execution will be collected under some reports root in
 directory structure like: reports_root/ClassName/MethodName_suffix.extension

tests/ci/pytest_launcher.py

@@ -0,0 +1,107 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import os
+import subprocess
+import sys
+
+
+PYTEST_REPORT = os.environ.get("PYTEST_REPORT",
+                               ".test_results/pytest_results.html")
+TESTR_REPORT = "testr_results.html"
+
+
+def error(msg):
+    print(msg)
+    exit(1)
+
+
+def main(args):
+    parser = argparse.ArgumentParser(args[0])
+    parser.add_argument("discovery_path", metavar="<path>", type=str,
+                        help="Path to location of all tests.")
+    parser.add_argument("--posargs", metavar="<str>", type=str, default="",
+                        help="TOX posargs. Currently supported only string to "
+                             "partial test or tests group to launch.")
+    args = parser.parse_args(args[1:])
+
+    # We allow only one parameter - path to partial test or tests group
+    path = args.posargs
+    if len(path.split(" ")) > 1:
+        error("Wrong value of posargs. It should include only path to single "
+              "test or tests group to launch.")
+    # NOTE(andreykurilin): Previously, next format was supported:
+    #   tests.unit.test_osclients.SomeTestCase.some_method
+    # It is more simple and pythonic than native pytest-way:
+    #   tests/unit/test_osclients.py::SomeTestCase::some_method
+    # Let's return this support
+    if path:
+        if "/" not in path:
+            path = path.split(".")
+            module = ""
+            for i in range(0, len(path)):
+                part = os.path.join(module, path[i])
+                if os.path.exists(part):
+                    module = part
+                    continue
+                if os.path.exists("%s.py" % part):
+                    if i != (len(path) - 1):
+                        module = "%s.py::%s" % (part, "::".join(path[i + 1:]))
+                    else:
+                        module = "%s.py" % part
+                    break
+
+                error("Non-existing path to single test or tests group to "
+                      "launch. %s %s" % (module, part))
+            path = module
+
+        path = os.path.abspath(os.path.expanduser(path))
+        if not path.startswith(os.path.abspath(args.discovery_path)):
+            # Prevent to launch functional tests from unit tests launcher.
+            error("Wrong path to single test or tests group to launch. It "
+                  "should be in %s." % args.discovery_path)
+    else:
+        path = args.discovery_path
+
+    print("Test(s) to launch (pytest format): %s" % path)
+
+    # NOTE(andreykurilin): we cannot publish pytest reports at gates, but we
+    #   can mask them as testr reports. It looks like a dirty hack and I
+    #   prefer to avoid it, but I see no other solutions at this point.
+
+    # apply dirty hack only in gates.
+    if os.environ.get("ZUUL_PROJECT"):
+        pytest_report = TESTR_REPORT
+    else:
+        pytest_report = PYTEST_REPORT
+
+    try:
+        subprocess.check_call(["py.test", "--html=%s" % pytest_report,
+                              "--durations=10", "-n", "auto", path],
+                              stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError:
+        # NOTE(andreykurilin): it is ok, since tests can fail.
+        exit_code = 1
+    else:
+        exit_code = 0
+
+    if os.path.exists(pytest_report) and os.environ.get("ZUUL_PROJECT"):
+        subprocess.check_call(["gzip", "-9", "-f", pytest_report],
+                              stderr=subprocess.STDOUT)
+
+    if exit_code == 1:
+        error("")
+
+if __name__ == "__main__":
+    main(sys.argv)

tests/unit/test_pytest_launcher.py

@@ -0,0 +1,76 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+import mock
+
+from tests.ci import pytest_launcher
+from tests.unit import test
+
+
+PATH = "tests.ci.pytest_launcher"
+
+
+class ExitError(Exception):
+    pass
+
+
+class PyTestLauncherTestCase(test.TestCase):
+    def setUp(self):
+        super(PyTestLauncherTestCase, self).setUp()
+
+        sp_patcher = mock.patch("%s.subprocess" % PATH)
+        self.sp = sp_patcher.start()
+        self.addCleanup(sp_patcher.stop)
+
+        exit_patcher = mock.patch("%s.exit" % PATH, side_effect=ExitError)
+        self.exit = exit_patcher.start()
+        self.addCleanup(exit_patcher.stop)
+
+        os_patcher = mock.patch("%s.os" % PATH)
+        self.os = os_patcher.start()
+        self.addCleanup(os_patcher.stop)
+        # emulate local run by default
+        self.os.environ = {}
+        self.os.path.join.side_effect = os.path.join
+        self.os.path.abspath.side_effect = os.path.abspath
+        self.os.path.expanduser.side_effect = os.path.expanduser
+
+    def test_wrong_posargs(self):
+        self.assertRaises(ExitError, pytest_launcher.main,
+                          ["script name", "test_path",
+                           "--posargs='posargs with spaces'"])
+
+        self.assertFalse(self.sp.called)
+        self.assertFalse(self.os.called)
+
+    def test_parsing_path(self):
+        def os_path_exists(path):
+            dpath = "some/path/to/some/test"
+            return dpath.startswith(path) or path == "%s/module.py" % dpath
+
+        self.os.path.exists.side_effect = os_path_exists
+
+        pytest_launcher.main(
+            ["script_name", "some/path",
+             "--posargs=some.path.to.some.test.module.TestCase.test"])
+
+        expected_path = os.path.abspath(
+            "some/path/to/some/test/module.py::TestCase::test")
+
+        self.assertEqual(1, self.sp.check_call.call_count)
+        call_args_obj = self.sp.check_call.call_args_list[0]
+        call_args = call_args_obj[0]
+        self.assertEqual(expected_path, call_args[0][-1])

tox.ini

@@ -19,7 +19,7 @@ install_command = pip install -U {opts} {packages}
 usedevelop = True
 commands =
   find . -type f -name "*.pyc" -delete
-  py.test --html=.test_results/pytest_results.html --durations=10 -n auto "tests/unit" {posargs}
+  python {toxinidir}/tests/ci/pytest_launcher.py tests/unit --posargs={posargs}
 distribute = false
 basepython = python2.7
 passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
@@ -55,8 +55,7 @@ commands = oslo_debug_helper -t tests {posargs}
 sitepackages = True
 commands =
   find . -type f -name "*.pyc" -delete
-  py.test --html=pytest_results.html --durations=10 -n auto "tests/functional" {posargs}
-
+  python {toxinidir}/tests/ci/pytest_launcher.py "tests/functional" --posargs={posargs}
 
 [testenv:cover]
 commands = {toxinidir}/tests/ci/cover.sh {posargs}