Support generation of report from N task

Added ability to build one report from
  results of multiple tasks. Tasks may be passed as UUID or as previously
  saved json results. Also it may be mixed list of UUIDs and file pathes.

  Closes bug 1406585

Change-Id: I11f2ca3ee6b868b79f91a4fa95e6a9ea918a79b0
This commit is contained in:
Oleh Anufriiev 2014-12-23 16:52:19 +02:00
parent 8e8dcf9360
commit fc823a6922
6 changed files with 369 additions and 40 deletions

View File

@ -20,6 +20,7 @@ import inspect
import os
import sys
import jsonschema
from oslo.config import cfg
import six
@ -298,8 +299,8 @@ def run(argv, categories):
validate_deprecated_args(argv, fn)
ret = fn(*fn_args, **fn_kwargs)
return(ret)
except (IOError, TypeError, exceptions.DeploymentNotFound,
exceptions.TaskNotFound) as e:
except (IOError, TypeError, ValueError, exceptions.DeploymentNotFound,
exceptions.TaskNotFound, jsonschema.ValidationError) as e:
if logging.is_debug():
LOG.exception(e)
print(e)

View File

@ -21,6 +21,8 @@ import os
import pprint
import webbrowser
import jsonschema
from oslo.utils import uuidutils
import yaml
from rally import api
@ -322,7 +324,9 @@ class TaskCommands(object):
"""
results = map(lambda x: {"key": x["key"], "result": x["data"]["raw"],
"sla": x["data"]["sla"]},
"sla": x["data"]["sla"],
"load_duration": x["data"]["load_duration"],
"full_duration": x["data"]["full_duration"]},
objects.Task.get(task_id).get_results())
if results:
@ -384,33 +388,79 @@ class TaskCommands(object):
print(_("There are no tasks. To run a new task, use:"
"\trally task start"))
@cliutils.args('--uuid', type=str, dest='task_id', help='uuid of task')
@cliutils.args('--out', type=str, dest='out', required=False,
@cliutils.args("--tasks", dest="tasks", nargs="+",
help="uuids of tasks or json files with task results")
@cliutils.args('--out', type=str, dest='out', required=True,
help='Path to output file.')
@cliutils.args('--open', dest='open_it', action='store_true',
help='Open it in browser.')
@envutils.with_default_task_id
def report(self, task_id=None, out=None, open_it=False):
@cliutils.deprecated_args(
"--uuid", dest="tasks", nargs="+",
help="uuids of tasks or json files with task results")
@envutils.default_from_global("tasks", envutils.ENV_TASK, "--uuid")
def report(self, tasks=None, out=None, open_it=False):
"""Generate HTML report file for specified task.
:param task_id: int, task identifier
:param task_id: UUID, task identifier
:param tasks: list, UUIDs od tasks or pathes files with tasks results
:param out: str, output html file name
:param open_it: bool, whether to open output file in web browser
"""
results = map(lambda x: {"key": x["key"],
"sla": x["data"]["sla"],
"result": x["data"]["raw"],
"load_duration": x["data"]["load_duration"],
"full_duration": x["data"]["full_duration"]},
objects.Task.get(task_id).get_results())
if out:
out = os.path.expanduser(out)
output_file = out or ("%s.html" % task_id)
tasks = isinstance(tasks, list) and tasks or [tasks]
results = list()
processed_names = dict()
for task_file_or_uuid in tasks:
if os.path.exists(os.path.expanduser(task_file_or_uuid)):
with open(os.path.expanduser(task_file_or_uuid),
"r") as inp_js:
tasks_results = json.load(inp_js)
for result in tasks_results:
try:
jsonschema.validate(
result,
objects.task.TASK_RESULT_SCHEMA)
except jsonschema.ValidationError as e:
msg = _("ERROR: Invalid task result format in %s"
) % task_file_or_uuid
print(msg)
if logging.is_debug():
print(e)
else:
print(e.message)
return 1
elif uuidutils.is_uuid_like(task_file_or_uuid):
tasks_results = map(lambda x: {"key": x["key"],
"sla": x["data"]["sla"],
"result": x["data"]["raw"],
"load_duration": x["data"][
"load_duration"],
"full_duration": x["data"][
"full_duration"]},
objects.Task.get(
task_file_or_uuid).get_results())
else:
print(_("ERROR: Invalid UUID or file name passed: %s"
) % task_file_or_uuid)
return 1
for task_result in tasks_results:
if task_result["key"]["name"] in processed_names:
processed_names[task_result["key"]["name"]] += 1
task_result["key"]["pos"] = processed_names[
task_result["key"]["name"]]
else:
processed_names[task_result["key"]["name"]] = 0
results.append(task_result)
output_file = os.path.expanduser(out)
with open(output_file, "w+") as f:
f.write(plot.plot(results))
if open_it:
webbrowser.open_new_tab("file://" + os.path.realpath(output_file))
webbrowser.open_new_tab("file://" + os.path.realpath(out))
# NOTE(maretskiy): plot2html is deprecated by `report'
# and should be removed later

View File

@ -15,10 +15,95 @@
import json
from rally.common import utils as rutils
from rally import consts
from rally import db
TASK_RESULT_SCHEMA = {
"type": "object",
"$schema": rutils.JSON_SCHEMA,
"properties": {
"key": {
"type": "object",
"properties": {
"kw": {
"type": "object"
},
"name": {
"type": "string"
},
"pos": {
"type": "integer"
},
},
"required": ["kw", "name", "pos"]
},
"sla": {
"type": "array",
"items": {
"type": "object",
"properties": {
"criterion": {
"type": "string"
},
"detail": {
"type": "string"
},
"success": {
"type": "boolean"
}
}
}
},
"result": {
"type": "array",
"items": {
"type": "object",
"properties": {
"atomic_actions": {
"type": "object"
},
"duration": {
"type": "number"
},
"error": {
"type": "array"
},
"idle_duration": {
"type": "number"
},
"scenario_output": {
"type": "object",
"properties": {
"data": {
"type": "object"
},
"errors": {
"type": "string"
},
},
"required": ["data", "errors"]
},
},
"required": ["atomic_actions", "duration", "error",
"idle_duration", "scenario_output"]
},
"minItems": 1
},
"load_duration": {
"type": "number",
},
"full_duration": {
"type": "number",
},
},
"required": ["key", "sla", "result", "load_duration",
"full_duration"],
"additionalProperties": False
}
class Task(object):
"""Represents a task object."""

View File

@ -99,7 +99,7 @@ class TaskTestCase(unittest.TestCase):
def test_report_with_wrong_task_id(self):
rally = utils.Rally()
self.assertRaises(utils.RallyCmdError,
rally, "task report --uuid %s" % FAKE_TASK_UUID)
rally, "task report --tasks %s" % FAKE_TASK_UUID)
def test_sla_check_with_wrong_task_id(self):
rally = utils.Rally()
@ -111,7 +111,7 @@ class TaskTestCase(unittest.TestCase):
self.assertRaises(utils.RallyCmdError,
rally, "task status --uuid %s" % FAKE_TASK_UUID)
def test_report(self):
def test_report_one_uuid(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
@ -122,7 +122,70 @@ class TaskTestCase(unittest.TestCase):
rally("task report --out %s" % html_file)
self.assertTrue(os.path.exists(html_file))
self.assertRaises(utils.RallyCmdError,
rally, "task report --uuid %s" % FAKE_TASK_UUID)
rally, "task report --report %s" % FAKE_TASK_UUID)
def test_report_bunch_uuids(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
html_file = "/tmp/test_plot.html"
if os.path.exists(html_file):
os.remove(html_file)
task_uuids = list()
for i in range(3):
res = rally("task start --task %s" % config.filename)
for line in res.splitlines():
if "finished" in line:
task_uuids.append(line.split(" ")[1])
rally("task report --tasks %s --out %s" % (" ".join(task_uuids),
html_file))
self.assertTrue(os.path.exists(html_file))
def test_report_bunch_files(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
html_file = "/tmp/test_plot.html"
if os.path.exists(html_file):
os.remove(html_file)
files = list()
for i in range(3):
rally("task start --task %s" % config.filename)
path = "/tmp/task_%d.html" % i
files.append(path)
with open(path, "w") as tr:
tr.write(rally("task results"))
rally("task report --tasks %s --out %s" % (" ".join(files),
html_file))
self.assertTrue(os.path.exists(html_file))
def test_report_one_uuid_one_file(self):
rally = utils.Rally()
cfg = self._get_sample_task_config()
config = utils.TaskConfig(cfg)
html_file = "/tmp/test_plot.html"
rally("task start --task %s" % config.filename)
if os.path.exists(html_file):
os.remove(html_file)
task_result_file = "/tmp/report_42.json"
with open(task_result_file, "w") as res:
res.write(rally("task results"))
task_run_output = rally(
"task start --task %s" % config.filename).splitlines()
for line in task_run_output:
if "is finished" in line:
task_uuid = line.split(" ")[1]
break
else:
return 1
rally("task report --tasks"
" %s %s --out %s" % (task_result_file, task_uuid, html_file))
self.assertTrue(os.path.exists(html_file))
self.assertRaises(utils.RallyCmdError,
rally, "task report --report %s" % FAKE_TASK_UUID)
def test_delete(self):
rally = utils.Rally()

View File

@ -140,10 +140,14 @@ class TaskCommandsTestCase(test.TestCase):
def test_results(self, mock_get, mock_json):
task_id = "foo_task_id"
data = [
{"key": "foo_key", "data": {"raw": "foo_raw", "sla": []}}
{"key": "foo_key", "data": {"raw": "foo_raw", "sla": [],
"load_duration": "lo_duration",
"full_duration": "fu_duration"}}
]
result = map(lambda x: {"key": x["key"],
"result": x["data"]["raw"],
"load_duration": x["data"]["load_duration"],
"full_duration": x["data"]["full_duration"],
"sla": x["data"]["sla"]}, data)
mock_results = mock.Mock(return_value=data)
mock_get.return_value = mock.Mock(get_results=mock_results)
@ -163,21 +167,27 @@ class TaskCommandsTestCase(test.TestCase):
mock_get.assert_called_once_with(task_id)
self.assertEqual(1, result)
@mock.patch("rally.cmd.commands.task.jsonschema.validate",
return_value=None)
@mock.patch("rally.cmd.commands.task.os.path.realpath",
side_effect=lambda p: "realpath_%s" % p)
@mock.patch("rally.cmd.commands.task.open", create=True)
@mock.patch("rally.cmd.commands.task.plot")
@mock.patch("rally.cmd.commands.task.webbrowser")
@mock.patch("rally.cmd.commands.task.objects.Task.get")
def test_report(self, mock_get, mock_web, mock_plot, mock_open, mock_os):
task_id = "foo_task_id"
def test_report_one_uuid(self, mock_get, mock_web, mock_plot, mock_open,
mock_os, mock_validate):
task_id = "eb290c30-38d8-4c8f-bbcc-fc8f74b004ae"
data = [
{"key": "foo_key", "data": {"raw": "foo_raw", "sla": "foo_sla",
"load_duration": 1.1,
"full_duration": 1.2}},
{"key": "bar_key", "data": {"raw": "bar_raw", "sla": "bar_sla",
"load_duration": 2.1,
"full_duration": 2.2}}]
{"key": {"name": "test", "pos": 0},
"data": {"raw": "foo_raw", "sla": "foo_sla",
"load_duration": 0.1,
"full_duration": 1.2}},
{"key": {"name": "test", "pos": 0},
"data": {"raw": "bar_raw", "sla": "bar_sla",
"load_duration": 2.1,
"full_duration": 2.2}}]
results = map(lambda x: {"key": x["key"],
"result": x["data"]["raw"],
"sla": x["data"]["sla"],
@ -194,17 +204,10 @@ class TaskCommandsTestCase(test.TestCase):
def reset_mocks():
for m in mock_get, mock_web, mock_plot, mock_open:
m.reset_mock()
self.task.report(task_id)
mock_open.assert_called_once_with(task_id + ".html", "w+")
self.task.report(tasks=task_id, out="/tmp/%s.html" % task_id)
mock_open.assert_called_once_with("/tmp/%s.html" % task_id, "w+")
mock_plot.plot.assert_called_once_with(results)
mock_write.assert_called_once_with("html_report")
mock_get.assert_called_once_with(task_id)
reset_mocks()
self.task.report(task_id, out="bar.html")
mock_open.assert_called_once_with("bar.html", "w+")
mock_plot.plot.assert_called_once_with(results)
mock_write.assert_called_once_with("html_report")
mock_get.assert_called_once_with(task_id)
@ -213,6 +216,133 @@ class TaskCommandsTestCase(test.TestCase):
mock_web.open_new_tab.assert_called_once_with(
"file://realpath_spam.html")
@mock.patch("rally.cmd.commands.task.jsonschema.validate",
return_value=None)
@mock.patch("rally.cmd.commands.task.os.path.realpath",
side_effect=lambda p: "realpath_%s" % p)
@mock.patch("rally.cmd.commands.task.open", create=True)
@mock.patch("rally.cmd.commands.task.plot")
@mock.patch("rally.cmd.commands.task.webbrowser")
@mock.patch("rally.cmd.commands.task.objects.Task.get")
def test_report_bunch_uuids(self, mock_get, mock_web, mock_plot, mock_open,
mock_os, mock_validate):
tasks = ["eb290c30-38d8-4c8f-bbcc-fc8f74b004ae",
"eb290c30-38d8-4c8f-bbcc-fc8f74b004af"]
data = [
{"key": {"name": "test", "pos": 0},
"data": {"raw": "foo_raw", "sla": "foo_sla",
"load_duration": 0.1,
"full_duration": 1.2}},
{"key": {"name": "test", "pos": 0},
"data": {"raw": "bar_raw", "sla": "bar_sla",
"load_duration": 2.1,
"full_duration": 2.2}}]
results = list()
for task_uuid in tasks:
results.extend(map(lambda x: {"key": x["key"],
"result": x["data"]["raw"],
"sla": x["data"]["sla"],
"load_duration": x[
"data"]["load_duration"],
"full_duration": x[
"data"]["full_duration"]},
data))
mock_results = mock.Mock(return_value=data)
mock_get.return_value = mock.Mock(get_results=mock_results)
mock_plot.plot.return_value = "html_report"
mock_write = mock.Mock()
mock_open.return_value.__enter__.return_value = (
mock.Mock(write=mock_write))
def reset_mocks():
for m in mock_get, mock_web, mock_plot, mock_open:
m.reset_mock()
self.task.report(tasks=tasks, out="/tmp/1_test.html")
mock_open.assert_called_once_with("/tmp/1_test.html", "w+")
mock_plot.plot.assert_called_once_with(results)
mock_write.assert_called_once_with("html_report")
expected_get_calls = [mock.call(task) for task in tasks]
mock_get.assert_has_calls(expected_get_calls, any_order=True)
@mock.patch("rally.cmd.commands.task.json.load")
@mock.patch("rally.cmd.commands.task.os.path.exists", return_value=True)
@mock.patch("rally.cmd.commands.task.jsonschema.validate",
return_value=None)
@mock.patch("rally.cmd.commands.task.os.path.realpath",
side_effect=lambda p: "realpath_%s" % p)
@mock.patch("rally.cmd.commands.task.open", create=True)
@mock.patch("rally.cmd.commands.task.plot")
def test_report_one_file(self, mock_plot, mock_open, mock_os,
mock_validate, mock_path_exists, mock_json_load):
task_file = "/tmp/some_file.json"
data = [
{"key": {"name": "test", "pos": 0},
"data": {"raw": "foo_raw", "sla": "foo_sla",
"load_duration": 0.1,
"full_duration": 1.2}},
{"key": {"name": "test", "pos": 1},
"data": {"raw": "bar_raw", "sla": "bar_sla",
"load_duration": 2.1,
"full_duration": 2.2}}]
results = map(lambda x: {"key": x["key"],
"result": x["data"]["raw"],
"sla": x["data"]["sla"],
"load_duration": x["data"]["load_duration"],
"full_duration": x["data"]["full_duration"]},
data)
mock_plot.plot.return_value = "html_report"
mock_write = mock.Mock()
mock_read = mock.MagicMock(return_value=results)
mock_json_load.return_value = results
mock_open.return_value.__enter__.return_value = (
mock.Mock(write=mock_write, read=mock_read))
def reset_mocks():
for m in mock_plot, mock_open, mock_json_load, mock_validate:
m.reset_mock()
self.task.report(tasks=task_file, out="/tmp/1_test.html")
expected_open_calls = [mock.call(task_file, "r"),
mock.call("/tmp/1_test.html", "w+")]
mock_open.assert_has_calls(expected_open_calls, any_order=True)
mock_plot.plot.assert_called_once_with(results)
mock_write.assert_called_once_with("html_report")
@mock.patch("rally.cmd.commands.task.os.path.exists", return_value=True)
@mock.patch("rally.cmd.commands.task.json.load")
@mock.patch("rally.cmd.commands.task.open", create=True)
def test_report_exceptions(self, mock_open, mock_json_load,
mock_path_exists):
results = [
{"key": {"name": "test", "pos": 0},
"data": {"raw": "foo_raw", "sla": "foo_sla",
"load_duration": 0.1,
"full_duration": 1.2}}]
mock_write = mock.Mock()
mock_read = mock.MagicMock(return_value=results)
mock_json_load.return_value = results
mock_open.return_value.__enter__.return_value = (
mock.Mock(write=mock_write, read=mock_read))
ret = self.task.report(tasks="/tmp/task.json",
out="/tmp/tmp.hsml")
self.assertEqual(ret, 1)
for m in mock_open, mock_json_load:
m.reset_mock()
mock_path_exists.return_value = False
ret = self.task.report(tasks="/tmp/task.json",
out="/tmp/tmp.hsml")
self.assertEqual(ret, 1)
@mock.patch('rally.cmd.commands.task.common_cliutils.print_list')
@mock.patch('rally.cmd.commands.task.envutils.get_global',
return_value='123456789')

View File

@ -21,7 +21,7 @@ _rally()
OPTS["task_detailed"]="--uuid --iterations-data"
OPTS["task_list"]="--deployment --all-deployments --status"
OPTS["task_plot2html"]="--uuid --out --open"
OPTS["task_report"]="--uuid --out --open"
OPTS["task_report"]="--tasks --out --open"
OPTS["task_results"]="--uuid"
OPTS["task_sla_check"]="--uuid --json"
OPTS["task_start"]="--deployment --task --tag --no-use"