diff --git a/rally/benchmark/utils.py b/rally/benchmark/utils.py
index 10ab0113f0..ee1a90b80b 100644
--- a/rally/benchmark/utils.py
+++ b/rally/benchmark/utils.py
@@ -51,16 +51,15 @@ def parameterize_from_test_config(benchmark_name):
 
 
 def _run_test(args):
-    test_args = args[0]
-    proc_n = args[2]
-    os.environ['OSTF_CONFIG'] = args[1]
+    test_args, ostf_config, proc_n = args
+    os.environ['OSTF_CONFIG'] = ostf_config
 
     with utils.StdOutCapture() as out:
-        status = pytest.main(args=test_args)
+        status = pytest.main(test_args)
 
-    return {'msg': [line for line in out.getvalue().split('\n')
-                    if '===' not in line or line],
-            'status': status, 'proc_name': proc_n}
+    return {'msg': out.getvalue(),
+            'status': status,
+            'proc_name': proc_n}
 
 
 class Tester(object):
@@ -109,7 +108,7 @@ class Tester(object):
 
         :param test_args: Arguments to be passed to pytest, e.g.
                           ['--pyargs', 'fuel_health.tests.sanity']
-        :param test_args: The number of times the test should be launched
+        :param times: The number of times the test should be launched
         :param concurrent: The number of concurrent processed to be used while
                            launching the test
 
@@ -126,13 +125,7 @@ class Tester(object):
                               for n in xrange(times))
         pool = multiprocessing.Pool(concurrent)
         result_generator = pool.imap(_run_test, iterable_test_args)
-
-        results = {}
-        for result in result_generator:
-            results.update({result['proc_name']: result})
-            if 'Timeout' in result['msg'][-2]:
-                break
-
+        results = dict([(r['proc_name'], r) for r in result_generator])
         self._cleanup(self._cloud_config_path)
         return results
 
diff --git a/tests/benchmark/test_utils.py b/tests/benchmark/test_utils.py
index 7a0b84c1c3..0c81394058 100644
--- a/tests/benchmark/test_utils.py
+++ b/tests/benchmark/test_utils.py
@@ -27,7 +27,7 @@ from rally.benchmark import utils
 from rally import test
 
 
-def test_dummy():
+def test_dummy_1():
     pass
 
 
@@ -36,7 +36,7 @@ def test_dummy_2():
 
 
 def test_dummy_timeout():
-    time.sleep(5)
+    time.sleep(1.1)
 
 
 class UtilsTestCase(test.NoDBTestCase):
@@ -57,7 +57,7 @@ class UtilsTestCase(test.NoDBTestCase):
 
     def test_running_test(self):
         tester = utils.Tester(self.cloud_config_path)
-        test = ['./tests/benchmark/test_utils.py', '-k', 'test_dummy']
+        test = ['./tests/benchmark/test_utils.py', '-k', 'test_dummy_1']
         for (times, concurrent) in [(1, 1), (3, 2), (2, 3)]:
             results = tester.run(test, times=times, concurrent=concurrent)
             self.assertEqual(len(results), times)
@@ -67,7 +67,7 @@ class UtilsTestCase(test.NoDBTestCase):
     def test_running_multiple_tests(self):
         tester = utils.Tester(self.cloud_config_path)
         tests_dict = {
-            'test1': ['./tests/benchmark/test_utils.py', '-k', 'test_dummy'],
+            'test1': ['./tests/benchmark/test_utils.py', '-k', 'test_dummy_1'],
             'test2': ['./tests/benchmark/test_utils.py', '-k', 'test_dummy_2']
         }
         for test_results in tester.run_all(tests_dict):
@@ -96,8 +96,19 @@ class UtilsTestCase(test.NoDBTestCase):
         tests.benchmark_tests = old_benchmark_tests
 
     def test_tester_timeout(self):
+        tester = utils.Tester(self.cloud_config_path)
+        test = ['./tests/benchmark/test_utils.py', '-k',
+                'test_dummy_timeout', '--timeout', '1']
+        results = tester.run(test, times=2, concurrent=2)
+        for result in results.values():
+            self.assertTrue('Timeout' in result['msg'])
+            self.assertTrue(result['status'] != 0)
+
+    def test_tester_no_timeout(self):
         tester = utils.Tester(self.cloud_config_path)
         test = ['./tests/benchmark/test_utils.py', '-k',
                 'test_dummy_timeout', '--timeout', '2']
-        results = tester.run(test, times=10, concurrent=2)
-        self.assertFalse('Timeout' in results.values()[0]['msg'][-2])
+        results = tester.run(test, times=2, concurrent=2)
+        for result in results.values():
+            self.assertTrue('Timeout' not in result['msg'])
+            self.assertTrue(result['status'] == 0)
diff --git a/tox.ini b/tox.ini
index f892a23547..97ce34013f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,7 +12,7 @@ deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 install_command = pip install -U {opts} {packages}
 usedevelop = True
-commands = python setup.py testr --testr-args='{posargs}'
+commands = python setup.py testr --slowest --testr-args='{posargs}'
 distribute = false
 
 [testenv:pep8]