openstack/rally-openstack
Code Issues Proposed changes

Input task templates and task cmd cleanup

Browse Source 
Implement task templates based on jinja2.

This allow us to pass as a task jinja2 template
and it's argument via arguments --task-args and
--task-args-file that should be dict in JSON or YAML
presentations.

So now command looks like:

rally task start <file> --task-args <template-args-json-or-yaml> \
  --task-args-file <file-with-args-in-json-yaml>

If both --task-args and --task-args-file then file dict is updated
by task args file.

Extend rally CI performance job. Now we can set template args
via file with name: ${TASK}_args.yaml

Bonus:
* Better message on InvalidTask format
* Remove redudant catch of "keyboardinterrupt"
  it should be implement in different way.
* Replace ' -> " in rally.cmd.commands.task
  and tests.unit.cmd.commands.task
* Imporve a bit CLI messages on rally task start
* Remove old plot2html command (it's enough deprecated)
* Improve test coverage of rally/cmd/commands/task
* Fix rally/cmd/commands/validate return 1 if bad format
* Write errors to stderr (in whole cmd/commands/task.py)

Change-Id: I7dadf2986bb10407865bc73bb2fb8c96a5162d9a
This commit is contained in:
Boris Pavlovic 2015-01-03 04:30:39 +03:00
parent 0f55aeee32
commit 254c0d68e8
7 changed files with 80 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
rally-jobs/rally-neutron.yaml
View File

@ -1,3 +1,4 @@
{% set image_name = "^cirros.*uec$" %}
--- ---
Dummy.dummy: Dummy.dummy:
- -
@ -302,7 +303,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 4 times: 4
@ -321,7 +322,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 4 times: 4
@ -340,7 +341,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 4 times: 4
@ -362,7 +363,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
detailed: True detailed: True
runner: runner:
type: "constant" type: "constant"
@ -382,7 +383,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
actions: actions:
- -
hard_reboot: 1 hard_reboot: 1
@ -410,7 +411,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
volume_size: 1 volume_size: 1
runner: runner:
type: "constant" type: "constant"
@ -430,7 +431,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
volume_size: 1 volume_size: 1
runner: runner:
type: "constant" type: "constant"
@ -450,7 +451,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 3 times: 3
@ -469,7 +470,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
to_flavor: to_flavor:
name: "m1.small" name: "m1.small"
confirm: true confirm: true
@ -549,7 +550,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
security_group_count: 5 security_group_count: 5
rules_per_security_group: 5 rules_per_security_group: 5
runner: runner:
@ -658,7 +659,7 @@
args: args:
size: 1 size: 1
image: image:
name: "^cirros.*uec$" name: {{image_name}}
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
runner: runner:
@ -690,7 +691,7 @@
users_per_tenant: 1 users_per_tenant: 1
servers: servers:
image: image:
name: "^cirros.*uec$" name: {{image_name}}
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
servers_per_tenant: 2 servers_per_tenant: 2
@ -713,7 +714,7 @@
users_per_tenant: 1 users_per_tenant: 1
servers: servers:
image: image:
name: "^cirros.*uec$" name: {{image_name}}
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
servers_per_tenant: 1 servers_per_tenant: 1
@ -741,7 +742,7 @@
users_per_tenant: 1 users_per_tenant: 1
servers: servers:
image: image:
name: "^cirros.*uec$" name: {{image_name}}
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
servers_per_tenant: 2 servers_per_tenant: 2

32
rally-jobs/rally.yaml
View File

@ -746,7 +746,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 3 times: 3
@ -764,7 +764,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 3 times: 3
@ -785,7 +785,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
force_delete: true force_delete: true
runner: runner:
type: "constant" type: "constant"
@ -805,7 +805,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
detailed: True detailed: True
runner: runner:
type: "constant" type: "constant"
@ -835,7 +835,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
servers_per_tenant: 2 servers_per_tenant: 2
sla: sla:
failure_rate: failure_rate:
@ -847,7 +847,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
to_flavor: to_flavor:
name: "m1.small" name: "m1.small"
confirm: true confirm: true
@ -869,7 +869,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
actions: actions:
- -
hard_reboot: 1 hard_reboot: 1
@ -895,7 +895,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
volume_size: 1 volume_size: 1
runner: runner:
type: "constant" type: "constant"
@ -915,7 +915,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
volume_size: 1 volume_size: 1
runner: runner:
type: "constant" type: "constant"
@ -935,7 +935,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 2 times: 2
@ -954,7 +954,7 @@
flavor: flavor:
name: "^ram64$" name: "^ram64$"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
auto_assign_nics: false auto_assign_nics: false
runner: runner:
type: "constant" type: "constant"
@ -976,7 +976,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
runner: runner:
type: "constant" type: "constant"
times: 3 times: 3
@ -995,7 +995,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
fixed_network: "private" fixed_network: "private"
floating_network: "public" floating_network: "public"
use_floatingip: true use_floatingip: true
@ -1019,7 +1019,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
volume_args: volume_args:
size: 2 size: 2
fixed_network: "private" fixed_network: "private"
@ -1045,7 +1045,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
fixed_network: "private" fixed_network: "private"
use_floatingip: false use_floatingip: false
script: "/home/jenkins/.rally/extra/instance_dd_test.sh" script: "/home/jenkins/.rally/extra/instance_dd_test.sh"
@ -1122,7 +1122,7 @@
flavor: flavor:
name: "m1.tiny" name: "m1.tiny"
image: image:
name: "^cirros.*uec$" name: {{image_name}}
security_group_count: 5 security_group_count: 5
rules_per_security_group: 5 rules_per_security_group: 5
runner: runner:

3
rally-jobs/rally_args.yaml Normal file
View File

@ -0,0 +1,3 @@
---
image_name: "^cirros.*uec$"

1
requirements.txt
View File

@ -5,6 +5,7 @@ Babel>=1.3
decorator>=3.4.0 decorator>=3.4.0
fixtures>=0.3.14 fixtures>=0.3.14
iso8601>=0.1.9 iso8601>=0.1.9
Jinja2>=2.6 # BSD License (3 clause)
jsonschema>=2.0.0,<3.0.0 jsonschema>=2.0.0,<3.0.0
netaddr>=0.7.12 netaddr>=0.7.12
oslo.config>=1.6.0 # Apache-2.0 oslo.config>=1.6.0 # Apache-2.0

14
tests/ci/rally-gate.sh
View File

@ -21,7 +21,14 @@ if [ ! -d $RALLY_JOB_DIR ]; then
RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-jobs RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-jobs
fi fi
SCENARIO=${RALLY_JOB_DIR}/${RALLY_SCENARIO}.yaml BASE_FOR_TASK=${RALLY_JOB_DIR}/${RALLY_SCENARIO}
TASK=${BASE_FOR_TASK}.yaml
TASK_ARGS=""
if [ -f ${BASE_FOR_TASK}_args.yaml ]; then
TASK_ARGS=" --task-args-file ${BASE_FOR_TASK}_args.yaml"
fi
PLUGINS_DIR=${RALLY_JOB_DIR}/plugins PLUGINS_DIR=${RALLY_JOB_DIR}/plugins
EXTRA_DIR=${RALLY_JOB_DIR}/extra EXTRA_DIR=${RALLY_JOB_DIR}/extra
@ -48,12 +55,13 @@ rally show images
rally show networks rally show networks
rally show secgroups rally show secgroups
rally show keypairs rally show keypairs
rally -v task start --task $SCENARIO
rally -v task start --task $TASK $TASK_ARGS
mkdir -p rally-plot/extra mkdir -p rally-plot/extra
python $BASE/new/rally/rally/ui/utils.py render\ python $BASE/new/rally/rally/ui/utils.py render\
tests/ci/rally-gate/index.mako > rally-plot/extra/index.html tests/ci/rally-gate/index.mako > rally-plot/extra/index.html
cp $SCENARIO rally-plot/task.txt cp $TASK rally-plot/task.txt
tar -czf rally-plot/plugins.tar.gz -C $RALLY_PLUGINS_DIR . tar -czf rally-plot/plugins.tar.gz -C $RALLY_PLUGINS_DIR .
rally task report --out rally-plot/results.html rally task report --out rally-plot/results.html
gzip -9 rally-plot/results.html gzip -9 rally-plot/results.html

38
tests/functional/test_cli_task.py
View File

@ -91,11 +91,6 @@ class TaskTestCase(unittest.TestCase):
self.assertRaises(utils.RallyCmdError, self.assertRaises(utils.RallyCmdError,
rally, "task detailed --uuid %s" % FAKE_TASK_UUID) rally, "task detailed --uuid %s" % FAKE_TASK_UUID)
def test_plot2html_with_wrong_task_id(self):
rally = utils.Rally()
self.assertRaises(utils.RallyCmdError,
rally, "task plot2html --uuid %s" % FAKE_TASK_UUID)
def test_report_with_wrong_task_id(self): def test_report_with_wrong_task_id(self):
rally = utils.Rally() rally = utils.Rally()
self.assertRaises(utils.RallyCmdError, self.assertRaises(utils.RallyCmdError,
@ -131,12 +126,12 @@ class TaskTestCase(unittest.TestCase):
html_file = "/tmp/test_plot.html" html_file = "/tmp/test_plot.html"
if os.path.exists(html_file): if os.path.exists(html_file):
os.remove(html_file) os.remove(html_file)
task_uuids = list() task_uuids = []
for i in range(3): for i in range(3):
res = rally("task start --task %s" % config.filename) res = rally("task start --task %s" % config.filename)
for line in res.splitlines(): for line in res.splitlines():
if "finished" in line: if "finished" in line:
task_uuids.append(line.split(" ")[1]) task_uuids.append(line.split(" ")[1][:-1])
rally("task report --tasks %s --out %s" % (" ".join(task_uuids), rally("task report --tasks %s --out %s" % (" ".join(task_uuids),
html_file)) html_file))
self.assertTrue(os.path.exists(html_file)) self.assertTrue(os.path.exists(html_file))
@ -222,11 +217,12 @@ class TaskTestCase(unittest.TestCase):
deployment_id = envutils.get_global("RALLY_DEPLOYMENT") deployment_id = envutils.get_global("RALLY_DEPLOYMENT")
cfg = {"invalid": "config"} cfg = {"invalid": "config"}
config = utils.TaskConfig(cfg) config = utils.TaskConfig(cfg)
output = rally(("task validate --task %(task_file)s " self.assertRaises(utils.RallyCmdError,
"--deployment %(deployment_id)s") % rally,
{"task_file": config.filename, ("task validate --task %(task_file)s "
"deployment_id": deployment_id}) "--deployment %(deployment_id)s") %
self.assertIn("Task config is invalid", output) {"task_file": config.filename,
"deployment_id": deployment_id})
def test_start(self): def test_start(self):
rally = utils.Rally() rally = utils.Rally()
@ -239,7 +235,7 @@ class TaskTestCase(unittest.TestCase):
{"task_file": config.filename, {"task_file": config.filename,
"deployment_id": deployment_id}) "deployment_id": deployment_id})
result = re.search( result = re.search(
r"(?P<task_id>[0-9a-f\-]{36}) is started", output) r"(?P<task_id>[0-9a-f\-]{36}): started", output)
self.assertIsNotNone(result) self.assertIsNotNone(result)
# NOTE(oanufriev): Not implemented # NOTE(oanufriev): Not implemented
@ -283,14 +279,14 @@ class SLATestCase(unittest.TestCase):
rally("task start --task %s" % config.filename) rally("task start --task %s" % config.filename)
rally("task sla_check") rally("task sla_check")
expected = [ expected = [
{"benchmark": "KeystoneBasic.create_and_list_users", {"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_seconds_per_iteration", "criterion": "max_seconds_per_iteration",
"detail": mock.ANY, "detail": mock.ANY,
"pos": 0, "status": "PASS"}, "pos": 0, "status": "PASS"},
{"benchmark": "KeystoneBasic.create_and_list_users", {"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_failure_percent", "criterion": "max_failure_percent",
"detail": mock.ANY, "detail": mock.ANY,
"pos": 0, "status": "PASS"}, "pos": 0, "status": "PASS"},
] ]
data = rally("task sla_check --json", getjson=True) data = rally("task sla_check --json", getjson=True)
self.assertEqual(expected, data) self.assertEqual(expected, data)

20
tests/unit/rally_jobs/test_jobs.py
View File

@ -18,6 +18,7 @@ import traceback
import mock import mock
import yaml import yaml
from rally import api
from rally.benchmark import engine from rally.benchmark import engine
import rally.common.utils as rutils import rally.common.utils as rutils
from tests.unit import test from tests.unit import test
@ -38,9 +39,22 @@ class RallyJobsTestCase(test.TestCase):
with open(full_path) as task_file: with open(full_path) as task_file:
try: try:
task_config = yaml.safe_load(task_file.read()) args_file = os.path.join(
eng = engine.BenchmarkEngine(task_config, self.rally_jobs_path,
mock.MagicMock()) filename.rsplit(".", 1)[0] + "_args.yaml")
args = {}
if os.path.exists(args_file):
args = yaml.safe_load(open(args_file).read())
if not isinstance(args, dict):
raise TypeError(
"args file %s must be dict in yaml or json "
"presenatation" % args_file)
task = api.task_template_render(task_file.read(), **args)
task = yaml.safe_load(task)
eng = engine.BenchmarkEngine(task, mock.MagicMock())
eng.validate() eng.validate()
except Exception: except Exception:
print(traceback.format_exc()) print(traceback.format_exc())