openstack/rally-openstack
Code Issues Proposed changes

Publishing functional tests reports

Browse Source 
Change-Id: I4b7543e7244ecc776e6d0955c0428423cca3ad3c
This commit is contained in:
Oleh Anufriiev 2015-01-13 15:46:27 +02:00
parent a33b5b4d39
commit 89bd4aa865
4 changed files with 133 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
doc/source/contribute.rst
View File

@ -161,6 +161,15 @@ To run functional tests locally::
#NOTE: openrc file with OpenStack admin credentials
Output of every Rally execution will be collected under some reports root in
directiry structure like: reports_root/ClassName/MethodName_suffix.extension
This functionality implemented in tests.functional.utils.Rally.__call__ method.
Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file
path and name if you need. You can use it to publish html reports, generated
during tests.
Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is
'rally-cli-output-files'.
Rally CI scripts
^^^^^^^^^^^^^^^^

10
tests/README.rst
View File

@ -64,6 +64,16 @@ To run functional tests locally::
#NOTE: openrc file with OpenStack admin credentials
Output of every Rally execution will be collected under some reports root in
directiry structure like: reports_root/ClassName/MethodName_suffix.extension
This functionality implemented in tests.functional.utils.Rally.__call__ method.
Use 'gen_report_path' method of 'Rally' class to get automaticaly generated file
path and name if you need. You can use it to publish html reports, generated
during tests.
Reports root can be passed throw environment variable 'REPORTS_ROOT'. Default is
'rally-cli-output-files'.
Rally CI scripts
----------------

69
tests/functional/test_cli_task.py
View File

@ -110,12 +110,10 @@ class TaskTestCase(unittest.TestCase):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
html_file = "/tmp/test_plot.html"
rally("task start --task %s" % config.filename)
if os.path.exists(html_file):
os.remove(html_file)
rally("task report --out %s" % html_file)
self.assertTrue(os.path.exists(html_file))
rally("task report --out %s" % rally.gen_report_path(extension="html"))
self.assertTrue(os.path.exists(
rally.gen_report_path(extension="html")))
self.assertRaises(utils.RallyCmdError,
rally, "task report --report %s" % FAKE_TASK_UUID)
@ -123,62 +121,59 @@ class TaskTestCase(unittest.TestCase):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
html_file = "/tmp/test_plot.html"
if os.path.exists(html_file):
os.remove(html_file)
task_uuids = []
task_uuids = list()
for i in range(3):
res = rally("task start --task %s" % config.filename)
for line in res.splitlines():
if "finished" in line:
task_uuids.append(line.split(" ")[1][:-1])
rally("task report --tasks %s --out %s" % (" ".join(task_uuids),
html_file))
self.assertTrue(os.path.exists(html_file))
rally("task report --tasks %s --out %s" % (
" ".join(task_uuids), rally.gen_report_path(extension="html")))
self.assertTrue(os.path.exists(
rally.gen_report_path(extension="html")))
def test_report_bunch_files(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
html_file = "/tmp/test_plot.html"
if os.path.exists(html_file):
os.remove(html_file)
files = list()
for i in range(3):
rally("task start --task %s" % config.filename)
path = "/tmp/task_%d.html" % i
path = "/tmp/task_%d.json" % i
files.append(path)
with open(path, "w") as tr:
tr.write(rally("task results"))
if os.path.exists(path):
os.remove(path)
rally("task results", report_path=path, raw=True)
rally("task report --tasks %s --out %s" % (" ".join(files),
html_file))
self.assertTrue(os.path.exists(html_file))
rally("task report --tasks %s --out %s" % (
" ".join(files), rally.gen_report_path(extension="html")))
self.assertTrue(os.path.exists(
rally.gen_report_path(extension="html")))
def test_report_one_uuid_one_file(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
html_file = "/tmp/test_plot.html"
rally("task start --task %s" % config.filename)
if os.path.exists(html_file):
os.remove(html_file)
task_result_file = "/tmp/report_42.json"
with open(task_result_file, "w") as res:
res.write(rally("task results"))
if os.path.exists(task_result_file):
os.remove(task_result_file)
rally("task results", report_path=task_result_file, raw=True)
task_run_output = rally(
"task start --task %s" % config.filename).splitlines()
for line in task_run_output:
if "is finished" in line:
task_uuid = line.split(" ")[1]
if "finished" in line:
task_uuid = line.split(" ")[1][:-1]
break
else:
return 1
rally("task report --tasks"
" %s %s --out %s" % (task_result_file, task_uuid, html_file))
self.assertTrue(os.path.exists(html_file))
" %s %s --out %s" % (task_result_file, task_uuid,
rally.gen_report_path(extension="html")))
self.assertTrue(os.path.exists(
rally.gen_report_path(extension="html")))
self.assertRaises(utils.RallyCmdError,
rally, "task report --report %s" % FAKE_TASK_UUID)
@ -187,20 +182,30 @@ class TaskTestCase(unittest.TestCase):
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
rally("task list")
self.assertIn("finished", rally("task status"))
rally("task delete")
self.assertNotIn("finishe", rally("task list"))
self.assertNotIn("finished", rally("task list"))
def test_list(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
rally("task start --task %s" % config.filename)
self.assertIn("finished", rally("task list --deployment MAIN"))
self.assertIn("There are no tasks",
rally("task list --status failed"))
self.assertIn("finished", rally("task list --status finished"))
self.assertIn("deployment_name", rally("task list --all-deployments"))
self.assertIn(
"deployment_name", rally("task list --all-deployments"))
self.assertRaises(utils.RallyCmdError,
rally, "task list --status not_existing_status")

79
tests/functional/utils.py
View File

@ -15,6 +15,7 @@
from six.moves import configparser
import inspect
import json
import os
import pwd
@ -93,17 +94,91 @@ class Rally(object):
self.args = ["rally"]
subprocess.call(["rally-manage", "db", "recreate"])
self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE)
self.reports_root = os.environ.get("REPORTS_ROOT",
"rally-cli-output-files")
self._created_files = list()
self("deployment create --file %s --name MAIN" % DEPLOYMENT_FILE,
write_report=False)
def __del__(self):
shutil.rmtree(self.tmp_dir)
def __call__(self, cmd, getjson=False):
def gen_report_path(self, suffix=None, extension=None, keep_old=False):
"""Report file path/name modifier
:param suffix: suffix that will be appended to filename.
It will be appended before extension
:param extension: file extension.
:param keep_old: if True, previous reports will not be deleted,
but rename to 'nameSuffix.old*.extension'
:return: complete report name to write report
"""
caller_frame = inspect.currentframe().f_back
if caller_frame.f_code.co_name == "__call__":
caller_frame = caller_frame.f_back
method_name = caller_frame.f_code.co_name
test_object = caller_frame.f_locals["self"]
class_name = test_object.__class__.__name__
if not os.path.exists("%s/%s" % (self.reports_root, class_name)):
os.makedirs("%s/%s" % (self.reports_root, class_name))
suff = suffix or ""
ext = extension or "txt"
path = "%s/%s/%s%s.%s" % (self.reports_root, class_name,
method_name, suff, ext)
if path not in self._created_files:
if os.path.exists(path):
if not keep_old:
os.remove(path)
else:
path_list = path.split(".")
old_suff = "old"
path_list.insert(-1, old_suff)
new_path = ".".join(path_list)
count = 0
while os.path.exists(new_path):
count += 1
path_list[-2] = "old%d" % count
new_path = ".".join(path_list)
os.rename(path, new_path)
self._created_files.append(path)
return path
def __call__(self, cmd, getjson=False, report_path=None, raw=False,
suffix=None, extension=None, keep_old=False,
write_report=True):
"""Call rally in the shell
:param cmd: rally command
:param getjson: in cases, when rally prints JSON, you can catch output
deserialized
:param report_path: if present, rally command and its output will be
wretten to file with passed file name
:param raw: don't write command itself to report file. Only output
will be written
"""
if not isinstance(cmd, list):
cmd = cmd.split(" ")
try:
output = subprocess.check_output(self.args + cmd,
stderr=subprocess.STDOUT)
if write_report:
if not report_path:
report_path = self.gen_report_path(
suffix=suffix, extension=extension, keep_old=keep_old)
with open(report_path, "a") as rep:
if not raw:
rep.write("\n%s:\n" % " ".join(self.args + cmd))
rep.write("%s\n" % output)
if getjson:
return json.loads(output)
return output.decode("utf-8")