[ci] Split legacy-rally-dsvm-cli job

* Use empty deployment

  For functional tests we should not rely on existing deployment,
  it is not needed, so we stop doing it and use just empty deployment

* Move functional tests under separate `tox-functional` job

Change-Id: I1eda2d455cbf0479ae8aeb54ef32ab21d9ccdeb1

tests/check_samples/test_certification_task.py

@@ -19,12 +19,17 @@ import traceback
 import unittest
 
 import rally as rally_m
-from tests.functional import utils
+from tests.check_samples import utils
 
 
 class TestPreCreatedTasks(unittest.TestCase):
 
+    def test_check_success(self):
+        rally = utils.Rally()
+        rally("deployment check")
+
     def test_task_samples_is_valid(self):
+
         rally = utils.Rally()
         full_path = os.path.join(
             os.path.dirname(rally_m.__file__), os.pardir, "tasks", "openstack")

tests/check_samples/test_task_samples.py

@@ -30,7 +30,7 @@ from rally.common import broker
 from rally.common import yamlutils as yaml
 from rally import plugins
 from rally.plugins.openstack.context.keystone import users
-from tests.functional import utils
+from tests.check_samples import utils
 
 
 class TestTaskSamples(unittest.TestCase):

tests/check_samples/utils.py

@@ -0,0 +1,32 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import subprocess
+
+from tests.functional import utils
+
+
+DEPLOYMENT_FILE = "/tmp/rally_functests_main_deployment.json"
+
+
+class Rally(utils.Rally):
+
+    def __init__(self, force_new_db=False):
+        self._DEPLOYMENT_CREATE_ARGS = " --file %s" % DEPLOYMENT_FILE
+        if not os.path.exists(DEPLOYMENT_FILE):
+            subprocess.call(["rally", "--log-file", "/dev/null",
+                             "deployment", "config"],
+                            stdout=open(DEPLOYMENT_FILE, "w"))
+        super(Rally, self).__init__(force_new_db=force_new_db)

tests/ci/rally_cli_job.sh

@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-LOCAL_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-DB_CONNECTION="$(rally db show)"
-
-if [[ $DB_CONNECTION == sqlite* ]]; then
-    CONCURRENCY=0
-else
-    # in case of not sqlite db backends we cannot launch tests in parallel due
-    # to possible conflicts
-    CONCURRENCY=1
-    # currently, RCI_KEEP_DB variable is used to not create new databases per
-    # each test
-    export RCI_KEEP_DB=1
-fi
-
-
-python $LOCAL_DIR/pytest_launcher.py "tests/functional" --concurrency $CONCURRENCY
-
-python $LOCAL_DIR/pytest_launcher.py "tests/check_samples"

tests/functional/test_cli_deployment.py

@@ -67,7 +67,7 @@ class DeploymentTestCase(unittest.TestCase):
 
     def test_check_success(self):
         rally = utils.Rally()
-        self.assertTrue(rally("deployment check"))
+        rally("deployment check")
 
     def test_check_fail(self):
         rally = utils.Rally()

tests/functional/utils.py

@@ -26,8 +26,6 @@ import tempfile
 from oslo_utils import encodeutils
 from six.moves import configparser
 
-DEPLOYMENT_FILE = "/tmp/rally_functests_main_deployment.json"
-
 
 class RallyCliError(Exception):
 
@@ -72,12 +70,9 @@ class Rally(object):
         output = rally("deployment list")
 
     """
+    _DEPLOYMENT_CREATE_ARGS = ""
 
-    def __init__(self, fake=False, force_new_db=False):
+    def __init__(self, force_new_db=False):
-        if not os.path.exists(DEPLOYMENT_FILE):
-            subprocess.call(["rally", "--log-file", "/dev/null",
-                             "deployment", "config"],
-                            stdout=open(DEPLOYMENT_FILE, "w"))
 
         # NOTE(sskripnick): we should change home dir to avoid races
         # and do not touch any user files in ~/.rally
@@ -119,7 +114,7 @@ class Rally(object):
         self.reports_root = os.environ.get("REPORTS_ROOT",
                                            "rally-cli-output-files")
         self._created_files = []
-        self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE,
+        self("deployment create --name MAIN%s" % self._DEPLOYMENT_CREATE_ARGS,
              write_report=False)
 
     def __del__(self):

tox.ini

@@ -57,7 +57,7 @@ commands = oslo_debug_helper -t tests {posargs}
 sitepackages = True
 commands =
   find . -type f -name "*.pyc" -delete
-  {toxinidir}/tests/ci/rally_cli_job.sh
+  python {toxinidir}/tests/ci/pytest_launcher.py tests/check_samples --posargs={posargs}
 
 [testenv:functional]
 sitepackages = True

zuul.yaml

@@ -12,13 +12,25 @@
     run: tests/ci/playbooks/rally-install/run.yaml
     timeout: 1800
 
+- job:
+    name: tox-functional
+    parent: tox
+    description: |
+      Run functional tests for a Rally project under cPython version 2.7.
+      Uses tox with the ``functional`` environment.
+    vars:
+      tox_envlist: functional
+    timeout: 600
+
 - project:
     name: openstack/rally
     check:
       jobs:
         - rally-install-ubuntu-xenial
         - rally-install-centos-7
+        - tox-functional
     gate:
       jobs:
         - rally-install-ubuntu-xenial
         - rally-install-centos-7
+        - tox-functional