Add basic tools for integration tests

Add class for represent task config
Add integration tests for rally SLA
Generate html test results

Change-Id: Ifc77d1efad1456df0325fcb0d0d9dca0d9b46866

tests_ci/rally-integrated.sh

@@ -0,0 +1,12 @@
+#!/bin/sh -ex
+
+env
+
+mkdir -p .testrepository
+python -m subunit.run discover tests_ci > .testrepository/subunit.log
+EXIT_CODE=$?
+
+subunit2pyunit < .testrepository/subunit.log
+subunit-stats < .testrepository/subunit.log
+
+exit $EXIT_CODE

tests_ci/test_cli.py

@@ -14,7 +14,9 @@
 #    under the License.
 
 import ConfigParser
+import json
 import os
+import pwd
 import shutil
 import subprocess
 import tempfile
@@ -22,7 +24,6 @@ import unittest
 
 import mock
 
-
 """Test rally command line interface.
 
 This module is intended for running by OpenStack CI system.
@@ -32,7 +33,6 @@ To start tests manually please use
 
 """
 
-
 TEST_ENV = {
             "OS_USERNAME": "admin",
             "OS_PASSWORD": "admin",
@@ -54,6 +54,18 @@ class RallyCmdError(Exception):
         return "Code: %d Output: %s\n" % (self.code, self.output)
 
 
+class TaskConfig(object):
+
+    def __init__(self, config):
+        config_file = tempfile.NamedTemporaryFile(delete=False)
+        config_file.write(json.dumps(config))
+        config_file.close()
+        self.filename = config_file.name
+
+    def __del__(self):
+        os.unlink(self.filename)
+
+
 class Rally(object):
     """Create and represent separate rally installation.
 
@@ -66,26 +78,35 @@ class Rally(object):
     """
 
     def __init__(self):
+        # NOTE(sskripnick): we shoud change home dir to avoid races
+        # and do not touch any user files in ~/.rally
+        os.environ["HOME"] = pwd.getpwuid(os.getuid()).pw_dir
+        subprocess.call("rally deployment config > /tmp/.rd.json", shell=True)
         self.tmp_dir = tempfile.mkdtemp()
-        config_filename = os.path.join(self.tmp_dir, 'conf')
+        os.environ["HOME"] = self.tmp_dir
+        config_filename = os.path.join(self.tmp_dir, "conf")
         config = ConfigParser.RawConfigParser()
-        config.add_section('database')
+        config.add_section("database")
-        config.set('database', 'connection', 'sqlite:///%s/db' % self.tmp_dir)
+        config.set("database", "connection", "sqlite:///%s/db" % self.tmp_dir)
-        with open(config_filename, 'wb') as conf:
+        with open(config_filename, "wb") as conf:
             config.write(conf)
-        self.args = ['rally', '-d', '-v', '--config-file', config_filename]
+        self.args = ["rally", "--config-file", config_filename]
-        subprocess.call(['rally-manage', '--config-file', config_filename,
+        subprocess.call(["rally-manage", "--config-file", config_filename,
-                         'db', 'recreate'])
+                         "db", "recreate"])
+        self("deployment create --file /tmp/.rd.json --name MAIN")
 
     def __del__(self):
         shutil.rmtree(self.tmp_dir)
 
-    def __call__(self, cmd):
+    def __call__(self, cmd, getjson=False):
         if not isinstance(cmd, list):
             cmd = cmd.split(" ")
         try:
-            return subprocess.check_output(self.args + cmd,
+            output = subprocess.check_output(self.args + cmd,
                                              stderr=subprocess.STDOUT)
+            if getjson:
+                return json.loads(output)
+            return output
         except subprocess.CalledProcessError as e:
             raise RallyCmdError(e.returncode, e.output)
 
@@ -94,7 +115,54 @@ class DeploymentTestCase(unittest.TestCase):
 
     def test_create_fromenv_list_endpoint(self):
         rally = Rally()
-        with mock.patch.dict('os.environ', TEST_ENV):
+        with mock.patch.dict("os.environ", TEST_ENV):
             rally("deployment create --name t_create --fromenv")
-        self.assertIn('t_create', rally("deployment list"))
+        self.assertIn("t_create", rally("deployment list"))
-        self.assertIn(TEST_ENV['OS_AUTH_URL'], rally("deployment endpoint"))
+        self.assertIn(TEST_ENV["OS_AUTH_URL"], rally("deployment endpoint"))
+
+
+class SLATestCase(unittest.TestCase):
+
+    def _get_sample_task_config(self, max_seconds_per_iteration=4,
+                                max_failure_percent=0):
+        return {
+            "KeystoneBasic.create_and_list_users": [
+                {
+                    "args": {
+                        "name_length": 10
+                    },
+                    "runner": {
+                        "type": "constant",
+                        "times": 5,
+                        "concurrency": 5
+                    },
+                    "sla": {
+                        "max_seconds_per_iteration": max_seconds_per_iteration,
+                        "max_failure_percent": max_failure_percent,
+                    }
+                }
+            ]
+        }
+
+    def test_sla_fail(self):
+        rally = Rally()
+        cfg = self._get_sample_task_config(max_seconds_per_iteration=0.001)
+        config = TaskConfig(cfg)
+        rally("task start --task %s" % config.filename)
+        self.assertRaises(RallyCmdError, rally, "task sla_check")
+
+    def test_sla_success(self):
+        rally = Rally()
+        config = TaskConfig(self._get_sample_task_config())
+        rally("task start --task %s" % config.filename)
+        rally("task sla_check")
+        expected = [
+                {"benchmark": "KeystoneBasic.create_and_list_users",
+                 "criterion": "max_seconds_per_iteration",
+                 "pos": 0, "success": True},
+                {"benchmark": "KeystoneBasic.create_and_list_users",
+                 "criterion": "max_failure_percent",
+                 "pos": 0, "success": True},
+        ]
+        data = rally("task sla_check --json", getjson=True)
+        self.assertEqual(expected, data)

tox.ini

@@ -23,7 +23,8 @@ distribute = false
 commands = {posargs}
 
 [testenv:cli]
-commands = python -m unittest tests_ci.test_cli
+sitepackages = True
+commands = {toxinidir}/tests_ci/rally-integrated.sh
 
 [testenv:cover]
 commands = python setup.py testr --coverage --testr-args='{posargs}'