Input task templates and task cmd cleanup

Implement task templates based on jinja2. This allow us to pass as a task jinja2 template and it's argument via arguments --task-args and --task-args-file that should be dict in JSON or YAML presentations. So now command looks like: rally task start <file> --task-args <template-args-json-or-yaml> \ --task-args-file <file-with-args-in-json-yaml> If both --task-args and --task-args-file then file dict is updated by task args file. Extend rally CI performance job. Now we can set template args via file with name: ${TASK}_args.yaml Bonus: * Better message on InvalidTask format * Remove redudant catch of "keyboardinterrupt" it should be implement in different way. * Replace ' -> " in rally.cmd.commands.task and tests.unit.cmd.commands.task * Imporve a bit CLI messages on rally task start * Remove old plot2html command (it's enough deprecated) * Improve test coverage of rally/cmd/commands/task * Fix rally/cmd/commands/validate return 1 if bad format * Write errors to stderr (in whole cmd/commands/task.py) Change-Id: I7dadf2986bb10407865bc73bb2fb8c96a5162d9a
2015-01-03 04:30:39 +03:00 · 2015-01-03 04:30:39 +03:00 · 254c0d68e8
commit 254c0d68e8
parent 0f55aeee32
7 changed files with 80 additions and 57 deletions
--- a/rally-jobs/rally-neutron.yaml
+++ b/rally-jobs/rally-neutron.yaml
@ -1,3 +1,4 @@
+{% set image_name = "^cirros.*uec$" %}
 ---
  Dummy.dummy:
    -
@ -302,7 +303,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 4
@ -321,7 +322,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 4
@ -340,7 +341,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 4
@ -362,7 +363,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        detailed: True
      runner:
        type: "constant"
@ -382,7 +383,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        actions:
          -
            hard_reboot: 1
@ -410,7 +411,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        volume_size: 1
      runner:
        type: "constant"
@ -430,7 +431,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        volume_size: 1
      runner:
        type: "constant"
@ -450,7 +451,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 3
@ -469,7 +470,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        to_flavor:
            name: "m1.small"
        confirm: true
@ -549,7 +550,7 @@
        flavor:
          name: "m1.tiny"
        image:
-          name: "^cirros.*uec$"
+          name: {{image_name}}
        security_group_count: 5
        rules_per_security_group: 5
      runner:
@ -658,7 +659,7 @@
      args:
        size: 1
        image:
-          name: "^cirros.*uec$"
+          name: {{image_name}}
        flavor:
          name: "m1.tiny"
      runner:
@ -690,7 +691,7 @@
          users_per_tenant: 1
        servers:
          image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
          flavor:
            name: "m1.tiny"
          servers_per_tenant: 2
@ -713,7 +714,7 @@
          users_per_tenant: 1
        servers:
          image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
          flavor:
            name: "m1.tiny"
          servers_per_tenant: 1
@ -741,7 +742,7 @@
          users_per_tenant: 1
        servers:
          image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
          flavor:
            name: "m1.tiny"
          servers_per_tenant: 2
--- a/rally-jobs/rally.yaml
+++ b/rally-jobs/rally.yaml
@ -746,7 +746,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 3
@ -764,7 +764,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 3
@ -785,7 +785,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        force_delete: true
      runner:
        type: "constant"
@ -805,7 +805,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        detailed: True
      runner:
        type: "constant"
@ -835,7 +835,7 @@
          flavor:
              name: "m1.tiny"
          image:
-              name: "^cirros.*uec$"
+              name: {{image_name}}
          servers_per_tenant: 2
      sla:
        failure_rate:
@ -847,7 +847,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        to_flavor:
            name: "m1.small"
        confirm: true
@ -869,7 +869,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        actions:
          -
            hard_reboot: 1
@ -895,7 +895,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        volume_size: 1
      runner:
        type: "constant"
@ -915,7 +915,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        volume_size: 1
      runner:
        type: "constant"
@ -935,7 +935,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 2
@ -954,7 +954,7 @@
        flavor:
            name: "^ram64$"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
        auto_assign_nics: false
      runner:
        type: "constant"
@ -976,7 +976,7 @@
        flavor:
            name: "m1.tiny"
        image:
-            name: "^cirros.*uec$"
+            name: {{image_name}}
      runner:
        type: "constant"
        times: 3
@ -995,7 +995,7 @@
        flavor:
          name: "m1.tiny"
        image:
-          name: "^cirros.*uec$"
+          name: {{image_name}}
        fixed_network: "private"
        floating_network: "public"
        use_floatingip: true
@ -1019,7 +1019,7 @@
        flavor:
          name: "m1.tiny"
        image:
-          name: "^cirros.*uec$"
+          name: {{image_name}}
        volume_args:
            size: 2
        fixed_network: "private"
@ -1045,7 +1045,7 @@
        flavor:
          name: "m1.tiny"
        image:
-          name: "^cirros.*uec$"
+          name: {{image_name}}
        fixed_network: "private"
        use_floatingip: false
        script: "/home/jenkins/.rally/extra/instance_dd_test.sh"
@ -1122,7 +1122,7 @@
        flavor:
          name: "m1.tiny"
        image:
-          name: "^cirros.*uec$"
+          name: {{image_name}}
        security_group_count: 5
        rules_per_security_group: 5
      runner:
--- a/rally-jobs/rally_args.yaml
+++ b/rally-jobs/rally_args.yaml
@ -0,0 +1,3 @@
+---
+
+  image_name: "^cirros.*uec$"
--- a/requirements.txt
+++ b/requirements.txt
@ -5,6 +5,7 @@ Babel>=1.3
 decorator>=3.4.0
 fixtures>=0.3.14
 iso8601>=0.1.9
+Jinja2>=2.6  # BSD License (3 clause)
 jsonschema>=2.0.0,<3.0.0
 netaddr>=0.7.12
 oslo.config>=1.6.0  # Apache-2.0
--- a/tests/ci/rally-gate.sh
+++ b/tests/ci/rally-gate.sh
@ -21,7 +21,14 @@ if [ ! -d $RALLY_JOB_DIR ]; then
    RALLY_JOB_DIR=$BASE/new/$PROJECT/rally-jobs
 fi

-SCENARIO=${RALLY_JOB_DIR}/${RALLY_SCENARIO}.yaml
+BASE_FOR_TASK=${RALLY_JOB_DIR}/${RALLY_SCENARIO}
+
+TASK=${BASE_FOR_TASK}.yaml
+TASK_ARGS=""
+if [ -f ${BASE_FOR_TASK}_args.yaml ]; then
+    TASK_ARGS=" --task-args-file ${BASE_FOR_TASK}_args.yaml"
+fi
+
 PLUGINS_DIR=${RALLY_JOB_DIR}/plugins
 EXTRA_DIR=${RALLY_JOB_DIR}/extra

@ -48,12 +55,13 @@ rally show images
 rally show networks
 rally show secgroups
 rally show keypairs
-rally -v task start --task $SCENARIO
+
+rally -v task start --task $TASK $TASK_ARGS

 mkdir -p rally-plot/extra
 python $BASE/new/rally/rally/ui/utils.py render\
    tests/ci/rally-gate/index.mako > rally-plot/extra/index.html
-cp $SCENARIO rally-plot/task.txt
+cp $TASK rally-plot/task.txt
 tar -czf rally-plot/plugins.tar.gz -C $RALLY_PLUGINS_DIR .
 rally task report --out rally-plot/results.html
 gzip -9 rally-plot/results.html
--- a/tests/functional/test_cli_task.py
+++ b/tests/functional/test_cli_task.py
@ -91,11 +91,6 @@ class TaskTestCase(unittest.TestCase):
        self.assertRaises(utils.RallyCmdError,
                          rally, "task detailed --uuid %s" % FAKE_TASK_UUID)

-    def test_plot2html_with_wrong_task_id(self):
-        rally = utils.Rally()
-        self.assertRaises(utils.RallyCmdError,
-                          rally, "task plot2html --uuid %s" % FAKE_TASK_UUID)
-
    def test_report_with_wrong_task_id(self):
        rally = utils.Rally()
        self.assertRaises(utils.RallyCmdError,
@ -131,12 +126,12 @@ class TaskTestCase(unittest.TestCase):
        html_file = "/tmp/test_plot.html"
        if os.path.exists(html_file):
            os.remove(html_file)
-        task_uuids = list()
+        task_uuids = []
        for i in range(3):
            res = rally("task start --task %s" % config.filename)
            for line in res.splitlines():
                if "finished" in line:
-                    task_uuids.append(line.split(" ")[1])
+                    task_uuids.append(line.split(" ")[1][:-1])
        rally("task report --tasks %s --out %s" % (" ".join(task_uuids),
                                                   html_file))
        self.assertTrue(os.path.exists(html_file))
@ -222,11 +217,12 @@ class TaskTestCase(unittest.TestCase):
            deployment_id = envutils.get_global("RALLY_DEPLOYMENT")
        cfg = {"invalid": "config"}
        config = utils.TaskConfig(cfg)
-        output = rally(("task validate --task %(task_file)s "
-                        "--deployment %(deployment_id)s") %
-                       {"task_file": config.filename,
-                        "deployment_id": deployment_id})
-        self.assertIn("Task config is invalid", output)
+        self.assertRaises(utils.RallyCmdError,
+                          rally,
+                          ("task validate --task %(task_file)s "
+                           "--deployment %(deployment_id)s") %
+                          {"task_file": config.filename,
+                           "deployment_id": deployment_id})

    def test_start(self):
        rally = utils.Rally()
@ -239,7 +235,7 @@ class TaskTestCase(unittest.TestCase):
                           {"task_file": config.filename,
                            "deployment_id": deployment_id})
        result = re.search(
-            r"(?P<task_id>[0-9a-f\-]{36}) is started", output)
+            r"(?P<task_id>[0-9a-f\-]{36}): started", output)
        self.assertIsNotNone(result)

    # NOTE(oanufriev): Not implemented
@ -283,14 +279,14 @@ class SLATestCase(unittest.TestCase):
        rally("task start --task %s" % config.filename)
        rally("task sla_check")
        expected = [
-                {"benchmark": "KeystoneBasic.create_and_list_users",
-                 "criterion": "max_seconds_per_iteration",
-                 "detail": mock.ANY,
-                 "pos": 0, "status": "PASS"},
-                {"benchmark": "KeystoneBasic.create_and_list_users",
-                 "criterion": "max_failure_percent",
-                 "detail": mock.ANY,
-                 "pos": 0, "status": "PASS"},
+            {"benchmark": "KeystoneBasic.create_and_list_users",
+             "criterion": "max_seconds_per_iteration",
+             "detail": mock.ANY,
+             "pos": 0, "status": "PASS"},
+            {"benchmark": "KeystoneBasic.create_and_list_users",
+             "criterion": "max_failure_percent",
+             "detail": mock.ANY,
+             "pos": 0, "status": "PASS"},
        ]
        data = rally("task sla_check --json", getjson=True)
        self.assertEqual(expected, data)
--- a/tests/unit/rally_jobs/test_jobs.py
+++ b/tests/unit/rally_jobs/test_jobs.py
@ -18,6 +18,7 @@ import traceback
 import mock
 import yaml

+from rally import api
 from rally.benchmark import engine
 import rally.common.utils as rutils
 from tests.unit import test
@ -38,9 +39,22 @@ class RallyJobsTestCase(test.TestCase):

            with open(full_path) as task_file:
                try:
-                    task_config = yaml.safe_load(task_file.read())
-                    eng = engine.BenchmarkEngine(task_config,
-                                                 mock.MagicMock())
+                    args_file = os.path.join(
+                        self.rally_jobs_path,
+                        filename.rsplit(".", 1)[0] + "_args.yaml")
+
+                    args = {}
+                    if os.path.exists(args_file):
+                        args = yaml.safe_load(open(args_file).read())
+                        if not isinstance(args, dict):
+                            raise TypeError(
+                                "args file %s must be dict in yaml or json "
+                                "presenatation" % args_file)
+
+                    task = api.task_template_render(task_file.read(), **args)
+                    task = yaml.safe_load(task)
+
+                    eng = engine.BenchmarkEngine(task, mock.MagicMock())
                    eng.validate()
                except Exception:
                    print(traceback.format_exc())