openstack/rally-openstack
Code Issues Proposed changes

Input task templates and task cmd cleanup

Browse Source 
Implement task templates based on jinja2.

This allow us to pass as a task jinja2 template
and it's argument via arguments --task-args and
--task-args-file that should be dict in JSON or YAML
presentations.

So now command looks like:

rally task start <file> --task-args <template-args-json-or-yaml> \
  --task-args-file <file-with-args-in-json-yaml>

If both --task-args and --task-args-file then file dict is updated
by task args file.

Extend rally CI performance job. Now we can set template args
via file with name: ${TASK}_args.yaml

Bonus:
* Better message on InvalidTask format
* Remove redudant catch of "keyboardinterrupt"
  it should be implement in different way.
* Replace ' -> " in rally.cmd.commands.task
  and tests.unit.cmd.commands.task
* Imporve a bit CLI messages on rally task start
* Remove old plot2html command (it's enough deprecated)
* Improve test coverage of rally/cmd/commands/task
* Fix rally/cmd/commands/validate return 1 if bad format
* Write errors to stderr (in whole cmd/commands/task.py)

Change-Id: I7dadf2986bb10407865bc73bb2fb8c96a5162d9a
This commit is contained in:
Boris Pavlovic 2015-01-03 04:30:39 +03:00
parent 0f55aeee32
commit 254c0d68e8
7 changed files with 80 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
rally-jobs/rally-neutron.yaml
View File

@ -1,3 +1,4 @@
{% set image_name = "^cirros.*uec$" %}
---
Dummy.dummy:
-
@ -302,7 +303,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 4
@ -321,7 +322,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 4
@ -340,7 +341,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 4
@ -362,7 +363,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
detailed: True
runner:
type: "constant"
@ -382,7 +383,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
actions:
-
hard_reboot: 1
@ -410,7 +411,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
volume_size: 1
runner:
type: "constant"
@ -430,7 +431,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
volume_size: 1
runner:
type: "constant"
@ -450,7 +451,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 3
@ -469,7 +470,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
to_flavor:
name: "m1.small"
confirm: true
@ -549,7 +550,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
security_group_count: 5
rules_per_security_group: 5
runner:
@ -658,7 +659,7 @@
args:
size: 1
image:
name: "^cirros.*uec$"
name: {{image_name}}
flavor:
name: "m1.tiny"
runner:
@ -690,7 +691,7 @@
users_per_tenant: 1
servers:
image:
name: "^cirros.*uec$"
name: {{image_name}}
flavor:
name: "m1.tiny"
servers_per_tenant: 2
@ -713,7 +714,7 @@
users_per_tenant: 1
servers:
image:
name: "^cirros.*uec$"
name: {{image_name}}
flavor:
name: "m1.tiny"
servers_per_tenant: 1
@ -741,7 +742,7 @@
users_per_tenant: 1
servers:
image:
name: "^cirros.*uec$"
name: {{image_name}}
flavor:
name: "m1.tiny"
servers_per_tenant: 2

32
rally-jobs/rally.yaml
View File

@ -746,7 +746,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 3
@ -764,7 +764,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 3
@ -785,7 +785,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
force_delete: true
runner:
type: "constant"
@ -805,7 +805,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
detailed: True
runner:
type: "constant"
@ -835,7 +835,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
servers_per_tenant: 2
sla:
failure_rate:
@ -847,7 +847,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
to_flavor:
name: "m1.small"
confirm: true
@ -869,7 +869,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
actions:
-
hard_reboot: 1
@ -895,7 +895,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
volume_size: 1
runner:
type: "constant"
@ -915,7 +915,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
volume_size: 1
runner:
type: "constant"
@ -935,7 +935,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 2
@ -954,7 +954,7 @@
flavor:
name: "^ram64$"
image:
name: "^cirros.*uec$"
name: {{image_name}}
auto_assign_nics: false
runner:
type: "constant"
@ -976,7 +976,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
runner:
type: "constant"
times: 3
@ -995,7 +995,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
fixed_network: "private"
floating_network: "public"
use_floatingip: true
@ -1019,7 +1019,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
volume_args:
size: 2
fixed_network: "private"
@ -1045,7 +1045,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
fixed_network: "private"
use_floatingip: false
script: "/home/jenkins/.rally/extra/instance_dd_test.sh"
@ -1122,7 +1122,7 @@
flavor:
name: "m1.tiny"
image:
name: "^cirros.*uec$"
name: {{image_name}}
security_group_count: 5
rules_per_security_group: 5
runner:

3
rally-jobs/rally_args.yaml Normal file
View File

@ -0,0 +1,3 @@
---
image_name: "^cirros.*uec$"

1
requirements.txt
View File

@ -5,6 +5,7 @@ Babel>=1.3
decorator>=3.4.0
fixtures>=0.3.14
iso8601>=0.1.9
Jinja2>=2.6 # BSD License (3 clause)
jsonschema>=2.0.0,<3.0.0
netaddr>=0.7.12
oslo.config>=1.6.0 # Apache-2.0

14
tests/ci/rally-gate.sh
View File

@ -21,7 +21,14 @@ if [ ! -d $RALLY_JOB_DIR ]; then
RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-jobs
fi
SCENARIO=${RALLY_JOB_DIR}/${RALLY_SCENARIO}.yaml
BASE_FOR_TASK=${RALLY_JOB_DIR}/${RALLY_SCENARIO}
TASK=${BASE_FOR_TASK}.yaml
TASK_ARGS=""
if [ -f ${BASE_FOR_TASK}_args.yaml ]; then
TASK_ARGS=" --task-args-file ${BASE_FOR_TASK}_args.yaml"
fi
PLUGINS_DIR=${RALLY_JOB_DIR}/plugins
EXTRA_DIR=${RALLY_JOB_DIR}/extra
@ -48,12 +55,13 @@ rally show images
rally show networks
rally show secgroups
rally show keypairs
rally -v task start --task $SCENARIO
rally -v task start --task $TASK $TASK_ARGS
mkdir -p rally-plot/extra
python $BASE/new/rally/rally/ui/utils.py render\
tests/ci/rally-gate/index.mako > rally-plot/extra/index.html
cp $SCENARIO rally-plot/task.txt
cp $TASK rally-plot/task.txt
tar -czf rally-plot/plugins.tar.gz -C $RALLY_PLUGINS_DIR .
rally task report --out rally-plot/results.html
gzip -9 rally-plot/results.html

38
tests/functional/test_cli_task.py
View File

@ -91,11 +91,6 @@ class TaskTestCase(unittest.TestCase):
self.assertRaises(utils.RallyCmdError,
rally, "task detailed --uuid %s" % FAKE_TASK_UUID)
def test_plot2html_with_wrong_task_id(self):
rally = utils.Rally()
self.assertRaises(utils.RallyCmdError,
rally, "task plot2html --uuid %s" % FAKE_TASK_UUID)
def test_report_with_wrong_task_id(self):
rally = utils.Rally()
self.assertRaises(utils.RallyCmdError,
@ -131,12 +126,12 @@ class TaskTestCase(unittest.TestCase):
html_file = "/tmp/test_plot.html"
if os.path.exists(html_file):
os.remove(html_file)
task_uuids = list()
task_uuids = []
for i in range(3):
res = rally("task start --task %s" % config.filename)
for line in res.splitlines():
if "finished" in line:
task_uuids.append(line.split(" ")[1])
task_uuids.append(line.split(" ")[1][:-1])
rally("task report --tasks %s --out %s" % (" ".join(task_uuids),
html_file))
self.assertTrue(os.path.exists(html_file))
@ -222,11 +217,12 @@ class TaskTestCase(unittest.TestCase):
deployment_id = envutils.get_global("RALLY_DEPLOYMENT")
cfg = {"invalid": "config"}
config = utils.TaskConfig(cfg)
output = rally(("task validate --task %(task_file)s "
"--deployment %(deployment_id)s") %
{"task_file": config.filename,
"deployment_id": deployment_id})
self.assertIn("Task config is invalid", output)
self.assertRaises(utils.RallyCmdError,
rally,
("task validate --task %(task_file)s "
"--deployment %(deployment_id)s") %
{"task_file": config.filename,
"deployment_id": deployment_id})
def test_start(self):
rally = utils.Rally()
@ -239,7 +235,7 @@ class TaskTestCase(unittest.TestCase):
{"task_file": config.filename,
"deployment_id": deployment_id})
result = re.search(
r"(?P<task_id>[0-9a-f\-]{36}) is started", output)
r"(?P<task_id>[0-9a-f\-]{36}): started", output)
self.assertIsNotNone(result)
# NOTE(oanufriev): Not implemented
@ -283,14 +279,14 @@ class SLATestCase(unittest.TestCase):
rally("task start --task %s" % config.filename)
rally("task sla_check")
expected = [
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_seconds_per_iteration",
"detail": mock.ANY,
"pos": 0, "status": "PASS"},
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_failure_percent",
"detail": mock.ANY,
"pos": 0, "status": "PASS"},
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_seconds_per_iteration",
"detail": mock.ANY,
"pos": 0, "status": "PASS"},
{"benchmark": "KeystoneBasic.create_and_list_users",
"criterion": "max_failure_percent",
"detail": mock.ANY,
"pos": 0, "status": "PASS"},
]
data = rally("task sla_check --json", getjson=True)
self.assertEqual(expected, data)

20
tests/unit/rally_jobs/test_jobs.py
View File

@ -18,6 +18,7 @@ import traceback
import mock
import yaml
from rally import api
from rally.benchmark import engine
import rally.common.utils as rutils
from tests.unit import test
@ -38,9 +39,22 @@ class RallyJobsTestCase(test.TestCase):
with open(full_path) as task_file:
try:
task_config = yaml.safe_load(task_file.read())
eng = engine.BenchmarkEngine(task_config,
mock.MagicMock())
args_file = os.path.join(
self.rally_jobs_path,
filename.rsplit(".", 1)[0] + "_args.yaml")
args = {}
if os.path.exists(args_file):
args = yaml.safe_load(open(args_file).read())
if not isinstance(args, dict):
raise TypeError(
"args file %s must be dict in yaml or json "
"presenatation" % args_file)
task = api.task_template_render(task_file.read(), **args)
task = yaml.safe_load(task)
eng = engine.BenchmarkEngine(task, mock.MagicMock())
eng.validate()
except Exception:
print(traceback.format_exc())