Init benchmark framework

We add rally.benchmark module that contains Tester.
This is base of our benchmark framework. It allows us to easy write
test scenarios that could call any test N times simuntaneously and collect results

Add rally.utils stdout captures that allows us to caputre stdout and stderr in easy
way using "with" expression.

Add test for our benchmark base and rally.utils

rally/benchmark/benchmark.py

@@ -0,0 +1,66 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import multiprocessing
+import os
+import pytest
+
+from rally import utils
+
+
+class Tester(object):
+
+    def __init__(self, config_path):
+        self.config = os.path.abspath(config_path)
+        self.q = multiprocessing.Queue()
+        self.tests = {
+            'sanity': ['--pyargs', 'fuel_health.tests.sanity'],
+            'smoke': ['--pyargs', 'fuel_health.tests.smoke', '-k',
+                      '"not (test_007 or test_008 or test_009)"'],
+            'snapshot_test': ['--pyargs', 'fuel_health.tests.smoke', '-k',
+                              '"test_snapshot"']
+        }
+        self.cleanUp = 'fuel_health.tests'
+
+    def run(self, test_name, repeats=1):
+        res = {}
+        processes = {}
+        for i in xrange(repeats):
+            name = "test_{0}".format(i)
+            args = (self.tests[test_name], self.config, self.q, name)
+            processes[name] = multiprocessing.Process(name=name, args=args,
+                                                      target=Tester._run_test)
+            processes[name].start()
+        running = processes.keys()
+        while 1:
+            for process in running:
+                if not processes[process].is_alive():
+                    running.remove(process)
+                    item = self.q.get()
+                    res[item['proc_name']] = item
+            if not running:
+                break
+        return res
+
+    @staticmethod
+    def _run_test(test_name, path, queue, proc_name):
+        os.environ['OSTF_CONFIG'] = path
+        with utils.StdOutCapture() as out:
+            status = pytest.main(args=test_name)
+            msg = filter(lambda line: line and '===' not in line,
+                         out.getvalue().split('\n'))
+            queue.put({'msg': msg, 'status': status, 'proc_name': proc_name})

rally/utils.py

@@ -0,0 +1,43 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import StringIO
+import sys
+
+
+class StdOutCapture(object):
+    def __init__(self):
+        self.stdout = sys.stdout
+
+    def __enter__(self):
+        sys.stdout = StringIO.StringIO()
+        return sys.stdout
+
+    def __exit__(self, type, value, traceback):
+        sys.stdout = self.stdout
+
+
+class StdErrCapture(object):
+    def __init__(self):
+        self.stderr = sys.stderr
+
+    def __enter__(self):
+        sys.stderr = StringIO.StringIO()
+        return sys.stderr
+
+    def __exit__(self, type, value, traceback):
+        sys.stderr = self.stderr

requirements.txt

@@ -8,6 +8,7 @@ psutil
 SQLAlchemy>=0.7.8,<0.7.99
 sh
 six
+pytest
 
 -f http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3
 oslo.config>=1.2.0a3

tests/benchmark/test_benchmark.py

@@ -0,0 +1,35 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Tests for benchmarks."""
+
+from rally.benchmark import benchmark
+from rally import test
+
+
+def test_dummy():
+    pass
+
+
+class BenchmarkTestCase(test.NoDBTestCase):
+
+    def test_running_test(self):
+        tester = benchmark.Tester('')
+        tester.tests['test'] = ['./tests/benchmark/test_benchmark.py',
+                                '-k', 'test_dummy']
+        for result in tester.run('test', 3).itervalues():
+            self.assertEqual(result['status'], 0)

tests/test_utils.py

@@ -0,0 +1,48 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013: Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Test for Rally utils."""
+
+from __future__ import print_function
+
+import sys
+
+from rally import test
+from rally import utils
+
+
+class StdIOCaptureTestCase(test.NoDBTestCase):
+
+    def test_stdout_capture(self):
+        stdout = sys.stdout
+        messages = ['abcdef', 'defgaga']
+        with utils.StdOutCapture() as out:
+            for msg in messages:
+                print(msg)
+
+        self.assertEqual(out.getvalue().rstrip('\n').split('\n'), messages)
+        self.assertEqual(stdout, sys.stdout)
+
+    def test_stderr_capture(self):
+        stderr = sys.stderr
+        messages = ['abcdef', 'defgaga']
+        with utils.StdErrCapture() as err:
+            for msg in messages:
+                print(msg, file=sys.stderr)
+
+        self.assertEqual(err.getvalue().rstrip('\n').split('\n'), messages)
+        self.assertEqual(stderr, sys.stderr)