Publishing functional tests reports
Change-Id: I4b7543e7244ecc776e6d0955c0428423cca3ad3c
This commit is contained in:
Oleh Anufriiev
parent
a33b5b4d39
commit
89bd4aa865
@ -161,6 +161,15 @@ To run functional tests locally::
|
||||
|
||||
#NOTE: openrc file with OpenStack admin credentials
|
||||
|
||||
Output of every Rally execution will be collected under some reports root in
|
||||
directiry structure like: reports_root/ClassName/MethodName_suffix.extension
|
||||
This functionality implemented in tests.functional.utils.Rally.__call__ method.
|
||||
Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file
|
||||
path and name if you need. You can use it to publish html reports, generated
|
||||
during tests.
|
||||
Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is
|
||||
'rally-cli-output-files'.
|
||||
|
||||
Rally CI scripts
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
|
||||
@ -64,6 +64,16 @@ To run functional tests locally::
|
||||
|
||||
#NOTE: openrc file with OpenStack admin credentials
|
||||
|
||||
Output of every Rally execution will be collected under some reports root in
|
||||
directiry structure like: reports_root/ClassName/MethodName_suffix.extension
|
||||
This functionality implemented in tests.functional.utils.Rally.__call__ method.
|
||||
Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file
|
||||
path and name if you need. You can use it to publish html reports, generated
|
||||
during tests.
|
||||
Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is
|
||||
'rally-cli-output-files'.
|
||||
|
||||
|
||||
Rally CI scripts
|
||||
----------------
|
||||
|
||||
|
||||
@ -110,12 +110,10 @@ class TaskTestCase(unittest.TestCase):
|
||||
rally = utils.Rally()
|
||||
cfg = self._get_sample_task_config()
|
||||
config = utils.TaskConfig(cfg)
|
||||
html_file = "/tmp/test_plot.html"
|
||||
rally("task start --task %s" % config.filename)
|
||||
if os.path.exists(html_file):
|
||||
os.remove(html_file)
|
||||
rally("task report --out %s" % html_file)
|
||||
self.assertTrue(os.path.exists(html_file))
|
||||
rally("task report --out %s" % rally.gen_report_path(extension="html"))
|
||||
self.assertTrue(os.path.exists(
|
||||
rally.gen_report_path(extension="html")))
|
||||
self.assertRaises(utils.RallyCmdError,
|
||||
rally, "task report --report %s" % FAKE_TASK_UUID)
|
||||
|
||||
@ -123,62 +121,59 @@ class TaskTestCase(unittest.TestCase):
|
||||
rally = utils.Rally()
|
||||
cfg = self._get_sample_task_config()
|
||||
config = utils.TaskConfig(cfg)
|
||||
html_file = "/tmp/test_plot.html"
|
||||
if os.path.exists(html_file):
|
||||
os.remove(html_file)
|
||||
task_uuids = []
|
||||
task_uuids = list()
|
||||
for i in range(3):
|
||||
res = rally("task start --task %s" % config.filename)
|
||||
for line in res.splitlines():
|
||||
if "finished" in line:
|
||||
task_uuids.append(line.split(" ")[1][:-1])
|
||||
rally("task report --tasks %s --out %s" % (" ".join(task_uuids),
|
||||
html_file))
|
||||
self.assertTrue(os.path.exists(html_file))
|
||||
rally("task report --tasks %s --out %s" % (
|
||||
" ".join(task_uuids), rally.gen_report_path(extension="html")))
|
||||
self.assertTrue(os.path.exists(
|
||||
rally.gen_report_path(extension="html")))
|
||||
|
||||
def test_report_bunch_files(self):
|
||||
rally = utils.Rally()
|
||||
cfg = self._get_sample_task_config()
|
||||
config = utils.TaskConfig(cfg)
|
||||
html_file = "/tmp/test_plot.html"
|
||||
if os.path.exists(html_file):
|
||||
os.remove(html_file)
|
||||
files = list()
|
||||
for i in range(3):
|
||||
rally("task start --task %s" % config.filename)
|
||||
path = "/tmp/task_%d.html" % i
|
||||
path = "/tmp/task_%d.json" % i
|
||||
files.append(path)
|
||||
with open(path, "w") as tr:
|
||||
tr.write(rally("task results"))
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
rally("task results", report_path=path, raw=True)
|
||||
|
||||
rally("task report --tasks %s --out %s" % (" ".join(files),
|
||||
html_file))
|
||||
self.assertTrue(os.path.exists(html_file))
|
||||
rally("task report --tasks %s --out %s" % (
|
||||
" ".join(files), rally.gen_report_path(extension="html")))
|
||||
self.assertTrue(os.path.exists(
|
||||
rally.gen_report_path(extension="html")))
|
||||
|
||||
def test_report_one_uuid_one_file(self):
|
||||
rally = utils.Rally()
|
||||
cfg = self._get_sample_task_config()
|
||||
config = utils.TaskConfig(cfg)
|
||||
html_file = "/tmp/test_plot.html"
|
||||
rally("task start --task %s" % config.filename)
|
||||
if os.path.exists(html_file):
|
||||
os.remove(html_file)
|
||||
task_result_file = "/tmp/report_42.json"
|
||||
with open(task_result_file, "w") as res:
|
||||
res.write(rally("task results"))
|
||||
if os.path.exists(task_result_file):
|
||||
os.remove(task_result_file)
|
||||
rally("task results", report_path=task_result_file, raw=True)
|
||||
|
||||
task_run_output = rally(
|
||||
"task start --task %s" % config.filename).splitlines()
|
||||
for line in task_run_output:
|
||||
if "is finished" in line:
|
||||
task_uuid = line.split(" ")[1]
|
||||
if "finished" in line:
|
||||
task_uuid = line.split(" ")[1][:-1]
|
||||
break
|
||||
else:
|
||||
return 1
|
||||
|
||||
rally("task report --tasks"
|
||||
" %s %s --out %s" % (task_result_file, task_uuid, html_file))
|
||||
self.assertTrue(os.path.exists(html_file))
|
||||
" %s %s --out %s" % (task_result_file, task_uuid,
|
||||
rally.gen_report_path(extension="html")))
|
||||
self.assertTrue(os.path.exists(
|
||||
rally.gen_report_path(extension="html")))
|
||||
self.assertRaises(utils.RallyCmdError,
|
||||
rally, "task report --report %s" % FAKE_TASK_UUID)
|
||||
|
||||
@ -187,20 +182,30 @@ class TaskTestCase(unittest.TestCase):
|
||||
cfg = self._get_sample_task_config()
|
||||
config = utils.TaskConfig(cfg)
|
||||
rally("task start --task %s" % config.filename)
|
||||
|
||||
rally("task list")
|
||||
|
||||
self.assertIn("finished", rally("task status"))
|
||||
rally("task delete")
|
||||
self.assertNotIn("finishe", rally("task list"))
|
||||
|
||||
self.assertNotIn("finished", rally("task list"))
|
||||
|
||||
def test_list(self):
|
||||
rally = utils.Rally()
|
||||
cfg = self._get_sample_task_config()
|
||||
config = utils.TaskConfig(cfg)
|
||||
rally("task start --task %s" % config.filename)
|
||||
|
||||
self.assertIn("finished", rally("task list --deployment MAIN"))
|
||||
|
||||
self.assertIn("There are no tasks",
|
||||
rally("task list --status failed"))
|
||||
|
||||
self.assertIn("finished", rally("task list --status finished"))
|
||||
self.assertIn("deployment_name", rally("task list --all-deployments"))
|
||||
|
||||
self.assertIn(
|
||||
"deployment_name", rally("task list --all-deployments"))
|
||||
|
||||
self.assertRaises(utils.RallyCmdError,
|
||||
rally, "task list --status not_existing_status")
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
|
||||
from six.moves import configparser
|
||||
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
@ -93,17 +94,91 @@ class Rally(object):
|
||||
self.args = ["rally"]
|
||||
subprocess.call(["rally-manage", "db", "recreate"])
|
||||
|
||||
self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE)
|
||||
self.reports_root = os.environ.get("REPORTS_ROOT",
|
||||
"rally-cli-output-files")
|
||||
self._created_files = list()
|
||||
|
||||
self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE,
|
||||
write_report=False)
|
||||
|
||||
def __del__(self):
|
||||
shutil.rmtree(self.tmp_dir)
|
||||
|
||||
def __call__(self, cmd, getjson=False):
|
||||
def gen_report_path(self, suffix=None, extension=None, keep_old=False):
|
||||
"""Report file path/name modifier
|
||||
|
||||
:param suffix: suffix that will be appended to filename.
|
||||
It will be appended before extension
|
||||
:param extension: file extension.
|
||||
:param keep_old: if True, previous reports will not be deleted,
|
||||
but rename to 'nameSuffix.old*.extension'
|
||||
|
||||
:return: complete report name to write report
|
||||
"""
|
||||
caller_frame = inspect.currentframe().f_back
|
||||
if caller_frame.f_code.co_name == "__call__":
|
||||
caller_frame = caller_frame.f_back
|
||||
|
||||
method_name = caller_frame.f_code.co_name
|
||||
test_object = caller_frame.f_locals["self"]
|
||||
class_name = test_object.__class__.__name__
|
||||
|
||||
if not os.path.exists("%s/%s" % (self.reports_root, class_name)):
|
||||
os.makedirs("%s/%s" % (self.reports_root, class_name))
|
||||
|
||||
suff = suffix or ""
|
||||
ext = extension or "txt"
|
||||
path = "%s/%s/%s%s.%s" % (self.reports_root, class_name,
|
||||
method_name, suff, ext)
|
||||
|
||||
if path not in self._created_files:
|
||||
if os.path.exists(path):
|
||||
if not keep_old:
|
||||
os.remove(path)
|
||||
else:
|
||||
path_list = path.split(".")
|
||||
old_suff = "old"
|
||||
path_list.insert(-1, old_suff)
|
||||
new_path = ".".join(path_list)
|
||||
count = 0
|
||||
while os.path.exists(new_path):
|
||||
count += 1
|
||||
path_list[-2] = "old%d" % count
|
||||
new_path = ".".join(path_list)
|
||||
os.rename(path, new_path)
|
||||
|
||||
self._created_files.append(path)
|
||||
return path
|
||||
|
||||
def __call__(self, cmd, getjson=False, report_path=None, raw=False,
|
||||
suffix=None, extension=None, keep_old=False,
|
||||
write_report=True):
|
||||
"""Call rally in the shell
|
||||
|
||||
:param cmd: rally command
|
||||
:param getjson: in cases, when rally prints JSON, you can catch output
|
||||
deserialized
|
||||
:param report_path: if present, rally command and its output will be
|
||||
wretten to file with passed file name
|
||||
:param raw: don't write command itself to report file. Only output
|
||||
will be written
|
||||
"""
|
||||
|
||||
if not isinstance(cmd, list):
|
||||
cmd = cmd.split(" ")
|
||||
try:
|
||||
output = subprocess.check_output(self.args + cmd,
|
||||
stderr=subprocess.STDOUT)
|
||||
|
||||
if write_report:
|
||||
if not report_path:
|
||||
report_path = self.gen_report_path(
|
||||
suffix=suffix, extension=extension, keep_old=keep_old)
|
||||
with open(report_path, "a") as rep:
|
||||
if not raw:
|
||||
rep.write("\n%s:\n" % " ".join(self.args + cmd))
|
||||
rep.write("%s\n" % output)
|
||||
|
||||
if getjson:
|
||||
return json.loads(output)
|
||||
return output.decode("utf-8")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user