[ci] Add job to run functional tests over py3 env

Change-Id: Ia9e2d88976f06c20847729ca011e112f9bc53ff8

.zuul.d/python-jobs.yaml

@@ -37,6 +37,16 @@
     vars:
       tox_env: functional
 
+- job:
+    name: rally-tox-functional-py3
+    parent: rally-tox-base
+    description: |
+      Run test for rally project.
+
+      Uses tox with the ``functional-py3`` environment.
+    vars:
+      tox_env: functional-py3
+
 - job:
     name: rally-tox-self
     parent: rally-tox-base

.zuul.d/zuul.yaml

@@ -40,6 +40,7 @@
         - rally-tox-py37
         - rally-tox-cover
         - rally-tox-functional
+        - rally-tox-functional-py3
         - rally-database-migration
         - rally-install-ubuntu-xenial
         - rally-install-ubuntu-bionic
@@ -56,6 +57,7 @@
         - rally-tox-py37
         - rally-tox-cover
         - rally-tox-functional
+        - rally-tox-functional-py3
         - rally-database-migration
         - rally-install-ubuntu-xenial
         - rally-install-ubuntu-bionic

tests/ci/rally_functional_job.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-LOCAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-DB_CONNECTION="$(rally db show)"
-
-if [[ $DB_CONNECTION == sqlite* ]]; then
-    CONCURRENCY=0
-else
-    # in case of not sqlite db backends we cannot launch tests in parallel due
-    # to possible conflicts
-    CONCURRENCY=1
-    # currently, RCI_KEEP_DB variable is used to not create new databases per
-    # each test
-    export RCI_KEEP_DB=1
-fi
-
-
-python $LOCAL_DIR/pytest_launcher.py "tests/functional" --concurrency $CONCURRENCY --posargs=$1

tests/functional/test_cli_env.py

@@ -35,7 +35,7 @@ class EnvTestCase(unittest.TestCase):
         self.assertEqual({}, env_data["platforms"])
 
     def _create_spec(self, spec):
-        f = tempfile.NamedTemporaryFile(delete=False)
+        f = tempfile.NamedTemporaryFile(mode="w", delete=False)
 
         def unlink():
             os.unlink(f.name)

tests/functional/test_cli_functional.py

@@ -16,18 +16,30 @@
 import subprocess
 import unittest
 
+import six
+
+from rally.utils import encodeutils
+
 
 class CLITestCase(unittest.TestCase):
 
     def test_rally_cli(self):
         try:
-            output = subprocess.check_output(["rally"],
+            subprocess.check_output(["rally"], stderr=subprocess.STDOUT)
-                                             stderr=subprocess.STDOUT)
         except subprocess.CalledProcessError as e:
-            output = e.output
+            output = encodeutils.safe_decode(e.output)
-        self.assertIn("too few arguments", output)
+        else:
+            self.fail("It should ve non-zero exit code.")
+
+        # NOTE(andreykurilin): we should have the same errors...
+        if six.PY2:
+            self.assertIn("too few arguments", output)
+        else:
+            self.assertIn("the following arguments are required: category",
+                          output)
 
     def test_version_cli(self):
-        output = subprocess.check_output(["rally", "version"],
+        output = encodeutils.safe_decode(
-                                         stderr=subprocess.STDOUT)
+            subprocess.check_output(["rally", "version"],
+                                    stderr=subprocess.STDOUT))
         self.assertIn("Rally version:", output)

tox.ini

@@ -67,10 +67,16 @@ commands =
   echo "do nothing"
 
 [testenv:functional]
-sitepackages = True
+basepython = python2.7
 commands =
   find . -type f -name "*.pyc" -delete
-  {toxinidir}/tests/ci/rally_functional_job.sh {posargs}
+  python {toxinidir}/tests/ci/pytest_launcher.py tests/functional --posargs={posargs}
+
+[testenv:functional-py3]
+basepython = python3
+commands =
+  find . -type f -name "*.pyc" -delete
+  python {toxinidir}/tests/ci/pytest_launcher.py tests/functional --posargs={posargs}
 
 [testenv:cover]
 commands = {toxinidir}/tests/ci/cover.sh {posargs}