From 89bd4aa865126ee45df0da38c4f3579209dd5448 Mon Sep 17 00:00:00 2001 From: Oleh Anufriiev Date: Tue, 13 Jan 2015 15:46:27 +0200 Subject: [PATCH] Publishing functional tests reports Change-Id: I4b7543e7244ecc776e6d0955c0428423cca3ad3c --- doc/source/contribute.rst | 9 ++++ tests/README.rst | 10 ++++ tests/functional/test_cli_task.py | 69 ++++++++++++++------------- tests/functional/utils.py | 79 ++++++++++++++++++++++++++++++- 4 files changed, 133 insertions(+), 34 deletions(-) diff --git a/doc/source/contribute.rst b/doc/source/contribute.rst index 116501e9..d5971a72 100644 --- a/doc/source/contribute.rst +++ b/doc/source/contribute.rst @@ -161,6 +161,15 @@ To run functional tests locally:: #NOTE: openrc file with OpenStack admin credentials +Output of every Rally execution will be collected under some reports root in +directiry structure like: reports_root/ClassName/MethodName_suffix.extension +This functionality implemented in tests.functional.utils.Rally.__call__ method. +Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file +path and name if you need. You can use it to publish html reports, generated +during tests. +Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is +'rally-cli-output-files'. + Rally CI scripts ^^^^^^^^^^^^^^^^ diff --git a/tests/README.rst b/tests/README.rst index 9969cf40..b3361bbb 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -64,6 +64,16 @@ To run functional tests locally:: #NOTE: openrc file with OpenStack admin credentials +Output of every Rally execution will be collected under some reports root in +directiry structure like: reports_root/ClassName/MethodName_suffix.extension +This functionality implemented in tests.functional.utils.Rally.__call__ method. +Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file +path and name if you need. You can use it to publish html reports, generated +during tests. +Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is +'rally-cli-output-files'. + + Rally CI scripts ---------------- diff --git a/tests/functional/test_cli_task.py b/tests/functional/test_cli_task.py index 09550806..81db910d 100644 --- a/tests/functional/test_cli_task.py +++ b/tests/functional/test_cli_task.py @@ -110,12 +110,10 @@ class TaskTestCase(unittest.TestCase): rally = utils.Rally() cfg = self._get_sample_task_config() config = utils.TaskConfig(cfg) - html_file = "/tmp/test_plot.html" rally("task start --task %s" % config.filename) - if os.path.exists(html_file): - os.remove(html_file) - rally("task report --out %s" % html_file) - self.assertTrue(os.path.exists(html_file)) + rally("task report --out %s" % rally.gen_report_path(extension="html")) + self.assertTrue(os.path.exists( + rally.gen_report_path(extension="html"))) self.assertRaises(utils.RallyCmdError, rally, "task report --report %s" % FAKE_TASK_UUID) @@ -123,62 +121,59 @@ class TaskTestCase(unittest.TestCase): rally = utils.Rally() cfg = self._get_sample_task_config() config = utils.TaskConfig(cfg) - html_file = "/tmp/test_plot.html" - if os.path.exists(html_file): - os.remove(html_file) - task_uuids = [] + task_uuids = list() for i in range(3): res = rally("task start --task %s" % config.filename) for line in res.splitlines(): if "finished" in line: task_uuids.append(line.split(" ")[1][:-1]) - rally("task report --tasks %s --out %s" % (" ".join(task_uuids), - html_file)) - self.assertTrue(os.path.exists(html_file)) + rally("task report --tasks %s --out %s" % ( + " ".join(task_uuids), rally.gen_report_path(extension="html"))) + self.assertTrue(os.path.exists( + rally.gen_report_path(extension="html"))) def test_report_bunch_files(self): rally = utils.Rally() cfg = self._get_sample_task_config() config = utils.TaskConfig(cfg) - html_file = "/tmp/test_plot.html" - if os.path.exists(html_file): - os.remove(html_file) files = list() for i in range(3): rally("task start --task %s" % config.filename) - path = "/tmp/task_%d.html" % i + path = "/tmp/task_%d.json" % i files.append(path) - with open(path, "w") as tr: - tr.write(rally("task results")) + if os.path.exists(path): + os.remove(path) + rally("task results", report_path=path, raw=True) - rally("task report --tasks %s --out %s" % (" ".join(files), - html_file)) - self.assertTrue(os.path.exists(html_file)) + rally("task report --tasks %s --out %s" % ( + " ".join(files), rally.gen_report_path(extension="html"))) + self.assertTrue(os.path.exists( + rally.gen_report_path(extension="html"))) def test_report_one_uuid_one_file(self): rally = utils.Rally() cfg = self._get_sample_task_config() config = utils.TaskConfig(cfg) - html_file = "/tmp/test_plot.html" rally("task start --task %s" % config.filename) - if os.path.exists(html_file): - os.remove(html_file) task_result_file = "/tmp/report_42.json" - with open(task_result_file, "w") as res: - res.write(rally("task results")) + if os.path.exists(task_result_file): + os.remove(task_result_file) + rally("task results", report_path=task_result_file, raw=True) task_run_output = rally( "task start --task %s" % config.filename).splitlines() for line in task_run_output: - if "is finished" in line: - task_uuid = line.split(" ")[1] + if "finished" in line: + task_uuid = line.split(" ")[1][:-1] break else: return 1 rally("task report --tasks" - " %s %s --out %s" % (task_result_file, task_uuid, html_file)) - self.assertTrue(os.path.exists(html_file)) + " %s %s --out %s" % (task_result_file, task_uuid, + rally.gen_report_path(extension="html"))) + self.assertTrue(os.path.exists( + rally.gen_report_path(extension="html"))) self.assertRaises(utils.RallyCmdError, rally, "task report --report %s" % FAKE_TASK_UUID) @@ -187,20 +182,30 @@ class TaskTestCase(unittest.TestCase): cfg = self._get_sample_task_config() config = utils.TaskConfig(cfg) rally("task start --task %s" % config.filename) + + rally("task list") + self.assertIn("finished", rally("task status")) rally("task delete") - self.assertNotIn("finishe", rally("task list")) + + self.assertNotIn("finished", rally("task list")) def test_list(self): rally = utils.Rally() cfg = self._get_sample_task_config() config = utils.TaskConfig(cfg) rally("task start --task %s" % config.filename) + self.assertIn("finished", rally("task list --deployment MAIN")) + self.assertIn("There are no tasks", rally("task list --status failed")) + self.assertIn("finished", rally("task list --status finished")) - self.assertIn("deployment_name", rally("task list --all-deployments")) + + self.assertIn( + "deployment_name", rally("task list --all-deployments")) + self.assertRaises(utils.RallyCmdError, rally, "task list --status not_existing_status") diff --git a/tests/functional/utils.py b/tests/functional/utils.py index 88372a0f..0229ab54 100644 --- a/tests/functional/utils.py +++ b/tests/functional/utils.py @@ -15,6 +15,7 @@ from six.moves import configparser +import inspect import json import os import pwd @@ -93,17 +94,91 @@ class Rally(object): self.args = ["rally"] subprocess.call(["rally-manage", "db", "recreate"]) - self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE) + self.reports_root = os.environ.get("REPORTS_ROOT", + "rally-cli-output-files") + self._created_files = list() + + self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE, + write_report=False) def __del__(self): shutil.rmtree(self.tmp_dir) - def __call__(self, cmd, getjson=False): + def gen_report_path(self, suffix=None, extension=None, keep_old=False): + """Report file path/name modifier + + :param suffix: suffix that will be appended to filename. + It will be appended before extension + :param extension: file extension. + :param keep_old: if True, previous reports will not be deleted, + but rename to 'nameSuffix.old*.extension' + + :return: complete report name to write report + """ + caller_frame = inspect.currentframe().f_back + if caller_frame.f_code.co_name == "__call__": + caller_frame = caller_frame.f_back + + method_name = caller_frame.f_code.co_name + test_object = caller_frame.f_locals["self"] + class_name = test_object.__class__.__name__ + + if not os.path.exists("%s/%s" % (self.reports_root, class_name)): + os.makedirs("%s/%s" % (self.reports_root, class_name)) + + suff = suffix or "" + ext = extension or "txt" + path = "%s/%s/%s%s.%s" % (self.reports_root, class_name, + method_name, suff, ext) + + if path not in self._created_files: + if os.path.exists(path): + if not keep_old: + os.remove(path) + else: + path_list = path.split(".") + old_suff = "old" + path_list.insert(-1, old_suff) + new_path = ".".join(path_list) + count = 0 + while os.path.exists(new_path): + count += 1 + path_list[-2] = "old%d" % count + new_path = ".".join(path_list) + os.rename(path, new_path) + + self._created_files.append(path) + return path + + def __call__(self, cmd, getjson=False, report_path=None, raw=False, + suffix=None, extension=None, keep_old=False, + write_report=True): + """Call rally in the shell + + :param cmd: rally command + :param getjson: in cases, when rally prints JSON, you can catch output + deserialized + :param report_path: if present, rally command and its output will be + wretten to file with passed file name + :param raw: don't write command itself to report file. Only output + will be written + """ + if not isinstance(cmd, list): cmd = cmd.split(" ") try: output = subprocess.check_output(self.args + cmd, stderr=subprocess.STDOUT) + + if write_report: + if not report_path: + report_path = self.gen_report_path( + suffix=suffix, extension=extension, keep_old=keep_old) + with open(report_path, "a") as rep: + if not raw: + rep.write("\n%s:\n" % " ".join(self.args + cmd)) + rep.write("%s\n" % output) + if getjson: return json.loads(output) return output.decode("utf-8")