Fixed incorrect timeout tests

In this tests string "Timeout" was searched in wrong place.

Also test functions was renamed, and now we run only tests
we need, instead of all tests (see argument -k for pytest).

Also fixed some typos in docstrings and reduced sleep values.

Blueprint test-engine-utils

Change-Id: Ib89f6071fc5d3091ad0c0134a6009f366796cfa3

rally/benchmark/utils.py

@@ -51,16 +51,15 @@ def parameterize_from_test_config(benchmark_name):
 
 
 def _run_test(args):
-    test_args = args[0]
-    proc_n = args[2]
-    os.environ['OSTF_CONFIG'] = args[1]
+    test_args, ostf_config, proc_n = args
+    os.environ['OSTF_CONFIG'] = ostf_config
 
     with utils.StdOutCapture() as out:
-        status = pytest.main(args=test_args)
+        status = pytest.main(test_args)
 
-    return {'msg': [line for line in out.getvalue().split('\n')
-                    if '===' not in line or line],
-            'status': status, 'proc_name': proc_n}
+    return {'msg': out.getvalue(),
+            'status': status,
+            'proc_name': proc_n}
 
 
 class Tester(object):
@@ -109,7 +108,7 @@ class Tester(object):
 
         :param test_args: Arguments to be passed to pytest, e.g.
                           ['--pyargs', 'fuel_health.tests.sanity']
-        :param test_args: The number of times the test should be launched
+        :param times: The number of times the test should be launched
         :param concurrent: The number of concurrent processed to be used while
                            launching the test
 
@@ -126,13 +125,7 @@ class Tester(object):
                               for n in xrange(times))
         pool = multiprocessing.Pool(concurrent)
         result_generator = pool.imap(_run_test, iterable_test_args)
-
-        results = {}
-        for result in result_generator:
-            results.update({result['proc_name']: result})
-            if 'Timeout' in result['msg'][-2]:
-                break
-
+        results = dict([(r['proc_name'], r) for r in result_generator])
         self._cleanup(self._cloud_config_path)
         return results
 

tests/benchmark/test_utils.py

@@ -27,7 +27,7 @@ from rally.benchmark import utils
 from rally import test
 
 
-def test_dummy():
+def test_dummy_1():
     pass
 
 
@@ -36,7 +36,7 @@ def test_dummy_2():
 
 
 def test_dummy_timeout():
-    time.sleep(5)
+    time.sleep(1.1)
 
 
 class UtilsTestCase(test.NoDBTestCase):
@@ -57,7 +57,7 @@ class UtilsTestCase(test.NoDBTestCase):
 
     def test_running_test(self):
         tester = utils.Tester(self.cloud_config_path)
-        test = ['./tests/benchmark/test_utils.py', '-k', 'test_dummy']
+        test = ['./tests/benchmark/test_utils.py', '-k', 'test_dummy_1']
         for (times, concurrent) in [(1, 1), (3, 2), (2, 3)]:
             results = tester.run(test, times=times, concurrent=concurrent)
             self.assertEqual(len(results), times)
@@ -67,7 +67,7 @@ class UtilsTestCase(test.NoDBTestCase):
     def test_running_multiple_tests(self):
         tester = utils.Tester(self.cloud_config_path)
         tests_dict = {
-            'test1': ['./tests/benchmark/test_utils.py', '-k', 'test_dummy'],
+            'test1': ['./tests/benchmark/test_utils.py', '-k', 'test_dummy_1'],
             'test2': ['./tests/benchmark/test_utils.py', '-k', 'test_dummy_2']
         }
         for test_results in tester.run_all(tests_dict):
@@ -96,8 +96,19 @@ class UtilsTestCase(test.NoDBTestCase):
         tests.benchmark_tests = old_benchmark_tests
 
     def test_tester_timeout(self):
+        tester = utils.Tester(self.cloud_config_path)
+        test = ['./tests/benchmark/test_utils.py', '-k',
+                'test_dummy_timeout', '--timeout', '1']
+        results = tester.run(test, times=2, concurrent=2)
+        for result in results.values():
+            self.assertTrue('Timeout' in result['msg'])
+            self.assertTrue(result['status'] != 0)
+
+    def test_tester_no_timeout(self):
         tester = utils.Tester(self.cloud_config_path)
         test = ['./tests/benchmark/test_utils.py', '-k',
                 'test_dummy_timeout', '--timeout', '2']
-        results = tester.run(test, times=10, concurrent=2)
-        self.assertFalse('Timeout' in results.values()[0]['msg'][-2])
+        results = tester.run(test, times=2, concurrent=2)
+        for result in results.values():
+            self.assertTrue('Timeout' not in result['msg'])
+            self.assertTrue(result['status'] == 0)

tox.ini

@@ -12,7 +12,7 @@ deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 install_command = pip install -U {opts} {packages}
 usedevelop = True
-commands = python setup.py testr --testr-args='{posargs}'
+commands = python setup.py testr --slowest --testr-args='{posargs}'
 distribute = false
 
 [testenv:pep8]