Merge "Publishing functional tests reports"

doc/source/contribute.rst

@@ -161,6 +161,15 @@ To run functional tests locally::
 
   #NOTE: openrc file with OpenStack admin credentials
 
+Output of every Rally execution will be collected under some reports root in
+directiry structure like: reports_root/ClassName/MethodName_suffix.extension
+This functionality implemented in tests.functional.utils.Rally.__call__ method.
+Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file
+path and name if you need. You can use it to publish html reports, generated
+during tests.
+Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is
+'rally-cli-output-files'.
+
 Rally CI scripts
 ^^^^^^^^^^^^^^^^
 

tests/README.rst

@@ -64,6 +64,16 @@ To run functional tests locally::
 
   #NOTE: openrc file with OpenStack admin credentials
 
+Output of every Rally execution will be collected under some reports root in
+directiry structure like: reports_root/ClassName/MethodName_suffix.extension
+This functionality implemented in tests.functional.utils.Rally.__call__ method.
+Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file
+path and name if you need. You can use it to publish html reports, generated
+during tests.
+Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is
+'rally-cli-output-files'.
+
+
 Rally CI scripts
 ----------------
 

tests/functional/test_cli_task.py

@@ -110,12 +110,10 @@ class TaskTestCase(unittest.TestCase):
         rally = utils.Rally()
         cfg = self._get_sample_task_config()
         config = utils.TaskConfig(cfg)
-        html_file = "/tmp/test_plot.html"
         rally("task start --task %s" % config.filename)
-        if os.path.exists(html_file):
+        rally("task report --out %s" % rally.gen_report_path(extension="html"))
-            os.remove(html_file)
+        self.assertTrue(os.path.exists(
-        rally("task report --out %s" % html_file)
+            rally.gen_report_path(extension="html")))
-        self.assertTrue(os.path.exists(html_file))
         self.assertRaises(utils.RallyCmdError,
                           rally, "task report --report %s" % FAKE_TASK_UUID)
 
@@ -123,62 +121,59 @@ class TaskTestCase(unittest.TestCase):
         rally = utils.Rally()
         cfg = self._get_sample_task_config()
         config = utils.TaskConfig(cfg)
-        html_file = "/tmp/test_plot.html"
+        task_uuids = list()
-        if os.path.exists(html_file):
-            os.remove(html_file)
-        task_uuids = []
         for i in range(3):
             res = rally("task start --task %s" % config.filename)
             for line in res.splitlines():
                 if "finished" in line:
                     task_uuids.append(line.split(" ")[1][:-1])
-        rally("task report --tasks %s --out %s" % (" ".join(task_uuids),
+        rally("task report --tasks %s --out %s" % (
-                                                   html_file))
+              " ".join(task_uuids), rally.gen_report_path(extension="html")))
-        self.assertTrue(os.path.exists(html_file))
+        self.assertTrue(os.path.exists(
+            rally.gen_report_path(extension="html")))
 
     def test_report_bunch_files(self):
         rally = utils.Rally()
         cfg = self._get_sample_task_config()
         config = utils.TaskConfig(cfg)
-        html_file = "/tmp/test_plot.html"
-        if os.path.exists(html_file):
-            os.remove(html_file)
         files = list()
         for i in range(3):
             rally("task start --task %s" % config.filename)
-            path = "/tmp/task_%d.html" % i
+            path = "/tmp/task_%d.json" % i
             files.append(path)
-            with open(path, "w") as tr:
+            if os.path.exists(path):
-                tr.write(rally("task results"))
+                os.remove(path)
+            rally("task results", report_path=path, raw=True)
 
-        rally("task report --tasks %s --out %s" % (" ".join(files),
+        rally("task report --tasks %s --out %s" % (
-                                                   html_file))
+              " ".join(files), rally.gen_report_path(extension="html")))
-        self.assertTrue(os.path.exists(html_file))
+        self.assertTrue(os.path.exists(
+            rally.gen_report_path(extension="html")))
 
     def test_report_one_uuid_one_file(self):
         rally = utils.Rally()
         cfg = self._get_sample_task_config()
         config = utils.TaskConfig(cfg)
-        html_file = "/tmp/test_plot.html"
         rally("task start --task %s" % config.filename)
-        if os.path.exists(html_file):
-            os.remove(html_file)
         task_result_file = "/tmp/report_42.json"
-        with open(task_result_file, "w") as res:
+        if os.path.exists(task_result_file):
-            res.write(rally("task results"))
+            os.remove(task_result_file)
+        rally("task results", report_path=task_result_file, raw=True)
 
         task_run_output = rally(
             "task start --task %s" % config.filename).splitlines()
         for line in task_run_output:
-            if "is finished" in line:
+            if "finished" in line:
-                task_uuid = line.split(" ")[1]
+                task_uuid = line.split(" ")[1][:-1]
                 break
         else:
             return 1
 
         rally("task report --tasks"
-              " %s %s --out %s" % (task_result_file, task_uuid, html_file))
+              " %s %s --out %s" % (task_result_file, task_uuid,
-        self.assertTrue(os.path.exists(html_file))
+                                   rally.gen_report_path(extension="html")))
+        self.assertTrue(os.path.exists(
+            rally.gen_report_path(extension="html")))
         self.assertRaises(utils.RallyCmdError,
                           rally, "task report --report %s" % FAKE_TASK_UUID)
 
@@ -187,20 +182,30 @@ class TaskTestCase(unittest.TestCase):
         cfg = self._get_sample_task_config()
         config = utils.TaskConfig(cfg)
         rally("task start --task %s" % config.filename)
+
+        rally("task list")
+
         self.assertIn("finished", rally("task status"))
         rally("task delete")
-        self.assertNotIn("finishe", rally("task list"))
+
+        self.assertNotIn("finished", rally("task list"))
 
     def test_list(self):
         rally = utils.Rally()
         cfg = self._get_sample_task_config()
         config = utils.TaskConfig(cfg)
         rally("task start --task %s" % config.filename)
+
         self.assertIn("finished", rally("task list --deployment MAIN"))
+
         self.assertIn("There are no tasks",
                       rally("task list --status failed"))
+
         self.assertIn("finished", rally("task list --status finished"))
-        self.assertIn("deployment_name", rally("task list --all-deployments"))
+
+        self.assertIn(
+            "deployment_name", rally("task list --all-deployments"))
+
         self.assertRaises(utils.RallyCmdError,
                           rally, "task list --status not_existing_status")
 

tests/functional/utils.py

@@ -15,6 +15,7 @@
 
 from six.moves import configparser
 
+import inspect
 import json
 import os
 import pwd
@@ -93,17 +94,91 @@ class Rally(object):
             self.args = ["rally"]
             subprocess.call(["rally-manage", "db", "recreate"])
 
-        self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE)
+        self.reports_root = os.environ.get("REPORTS_ROOT",
+                                           "rally-cli-output-files")
+        self._created_files = list()
+
+        self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE,
+             write_report=False)
 
     def __del__(self):
         shutil.rmtree(self.tmp_dir)
 
-    def __call__(self, cmd, getjson=False):
+    def gen_report_path(self, suffix=None, extension=None, keep_old=False):
+        """Report file path/name modifier
+
+        :param suffix: suffix that will be appended to filename.
+            It will be appended before extension
+        :param extension: file extension.
+        :param keep_old: if True, previous reports will not be deleted,
+            but rename to 'nameSuffix.old*.extension'
+
+        :return: complete report name to write report
+        """
+        caller_frame = inspect.currentframe().f_back
+        if caller_frame.f_code.co_name == "__call__":
+            caller_frame = caller_frame.f_back
+
+        method_name = caller_frame.f_code.co_name
+        test_object = caller_frame.f_locals["self"]
+        class_name = test_object.__class__.__name__
+
+        if not os.path.exists("%s/%s" % (self.reports_root, class_name)):
+            os.makedirs("%s/%s" % (self.reports_root, class_name))
+
+        suff = suffix or ""
+        ext = extension or "txt"
+        path = "%s/%s/%s%s.%s" % (self.reports_root, class_name,
+                                  method_name, suff, ext)
+
+        if path not in self._created_files:
+            if os.path.exists(path):
+                if not keep_old:
+                    os.remove(path)
+                else:
+                    path_list = path.split(".")
+                    old_suff = "old"
+                    path_list.insert(-1, old_suff)
+                    new_path = ".".join(path_list)
+                    count = 0
+                    while os.path.exists(new_path):
+                        count += 1
+                        path_list[-2] = "old%d" % count
+                        new_path = ".".join(path_list)
+                    os.rename(path, new_path)
+
+            self._created_files.append(path)
+        return path
+
+    def __call__(self, cmd, getjson=False, report_path=None, raw=False,
+                 suffix=None, extension=None, keep_old=False,
+                 write_report=True):
+        """Call rally in the shell
+
+        :param cmd: rally command
+        :param getjson: in cases, when rally prints JSON, you can catch output
+            deserialized
+        :param report_path: if present, rally command and its output will be
+            wretten to file with passed file name
+        :param raw: don't write command itself to report file. Only output
+            will be written
+        """
+
         if not isinstance(cmd, list):
             cmd = cmd.split(" ")
         try:
             output = subprocess.check_output(self.args + cmd,
                                              stderr=subprocess.STDOUT)
+
+            if write_report:
+                if not report_path:
+                    report_path = self.gen_report_path(
+                        suffix=suffix, extension=extension, keep_old=keep_old)
+                with open(report_path, "a") as rep:
+                    if not raw:
+                        rep.write("\n%s:\n" % " ".join(self.args + cmd))
+                    rep.write("%s\n" % output)
+
             if getjson:
                 return json.loads(output)
             return output.decode("utf-8")