From d31279c2e44598e515ab96006ff92b1f864f7d76 Mon Sep 17 00:00:00 2001 From: Roman Vasilets Date: Thu, 25 Feb 2016 16:25:41 +0200 Subject: [PATCH] [Bug1535948] Add Heat template validator * add heat template validator to the validators. * add `template_validator` to each Heat scenario with usage of templates * add missed sla section to some scenarios in mos job * fix samples - allow them to work from the box Change-Id: I33aa709a3c4b94cb48c8067df685eee6442c0210 Closes-bug: #1535948 --- rally-jobs/rally-mos.yaml | 14 ++++++++++++++ rally/plugins/openstack/scenarios/heat/stacks.py | 7 +++++++ .../scenarios/heat/create-and-delete-stack.json | 2 +- .../scenarios/heat/create-and-delete-stack.yaml | 2 +- .../scenarios/heat/create-and-list-stack.json | 2 +- .../scenarios/heat/create-and-list-stack.yaml | 2 +- .../scenarios/heat/create-check-delete-stack.json | 2 +- .../scenarios/heat/create-check-delete-stack.yaml | 2 +- .../heat/create-snapshot-restore-delete-stack.json | 2 +- .../heat/create-snapshot-restore-delete-stack.yaml | 2 +- .../scenarios/heat/create-stack-and-scale.json | 2 +- .../scenarios/heat/create-stack-and-scale.yaml | 2 +- .../heat/create-suspend-resume-delete-stack.json | 2 +- .../heat/create-suspend-resume-delete-stack.yaml | 2 +- .../heat/create-update-delete-stack-add-res.json | 4 ++-- .../heat/create-update-delete-stack-add-res.yaml | 4 ++-- .../heat/create-update-delete-stack-del-res.json | 4 ++-- .../heat/create-update-delete-stack-del-res.yaml | 4 ++-- .../heat/create-update-delete-stack-increase.json | 4 ++-- .../heat/create-update-delete-stack-increase.yaml | 4 ++-- .../heat/create-update-delete-stack-inplace.json | 4 ++-- .../heat/create-update-delete-stack-inplace.yaml | 4 ++-- .../heat/create-update-delete-stack-reduce.json | 4 ++-- .../heat/create-update-delete-stack-reduce.yaml | 4 ++-- .../heat/create-update-delete-stack-replace.json | 4 ++-- .../heat/create-update-delete-stack-replace.yaml | 4 ++-- 26 files changed, 57 insertions(+), 36 deletions(-) diff --git a/rally-jobs/rally-mos.yaml b/rally-jobs/rally-mos.yaml index 49167f10..b1410e07 100644 --- a/rally-jobs/rally-mos.yaml +++ b/rally-jobs/rally-mos.yaml @@ -129,6 +129,8 @@ HeatStacks.create_and_list_stack: - + args: + template_path: "~/.rally/extra/default.yaml.template" runner: type: "constant" times: 1 @@ -137,9 +139,14 @@ users: tenants: 1 users_per_tenant: 1 + sla: + failure_rate: + max: 0 HeatStacks.create_and_delete_stack: - + args: + template_path: "~/.rally/extra/default.yaml.template" runner: type: "constant" times: 1 @@ -148,6 +155,9 @@ users: tenants: 1 users_per_tenant: 1 + sla: + failure_rate: + max: 0 Authenticate.keystone: - @@ -534,6 +544,10 @@ nova: security_groups: -1 security_group_rules: -1 + sla: + failure_rate: + max: 0 + NeutronNetworks.create_and_list_networks: diff --git a/rally/plugins/openstack/scenarios/heat/stacks.py b/rally/plugins/openstack/scenarios/heat/stacks.py index 1d0d13c7..1dd3e46d 100644 --- a/rally/plugins/openstack/scenarios/heat/stacks.py +++ b/rally/plugins/openstack/scenarios/heat/stacks.py @@ -25,6 +25,7 @@ class HeatStacks(utils.HeatScenario): """Benchmark scenarios for Heat stacks.""" @types.convert(template_path={"type": "file"}, files={"type": "file_dict"}) + @validation.validate_heat_template("template_path") @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @scenario.configure(context={"cleanup": ["heat"]}) @@ -56,6 +57,7 @@ class HeatStacks(utils.HeatScenario): self.clients("heat").resources.list(stack.id) @types.convert(template_path={"type": "file"}, files={"type": "file_dict"}) + @validation.validate_heat_template("template_path") @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @scenario.configure(context={"cleanup": ["heat"]}) @@ -77,6 +79,7 @@ class HeatStacks(utils.HeatScenario): self._delete_stack(stack) @types.convert(template_path={"type": "file"}, files={"type": "file_dict"}) + @validation.validate_heat_template("template_path") @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @scenario.configure(context={"cleanup": ["heat"]}) @@ -104,6 +107,7 @@ class HeatStacks(utils.HeatScenario): updated_template_path={"type": "file"}, files={"type": "file_dict"}, updated_files={"type": "file_dict"}) + @validation.validate_heat_template("template_path") @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @scenario.configure(context={"cleanup": ["heat"]}) @@ -139,6 +143,7 @@ class HeatStacks(utils.HeatScenario): self._delete_stack(stack) @types.convert(template_path={"type": "file"}, files={"type": "file_dict"}) + @validation.validate_heat_template("template_path") @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @scenario.configure(context={"cleanup": ["heat"]}) @@ -176,6 +181,7 @@ class HeatStacks(utils.HeatScenario): self._scale_stack(stack, output_key, delta) @types.convert(template_path={"type": "file"}, files={"type": "file_dict"}) + @validation.validate_heat_template("template_path") @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @scenario.configure(context={"cleanup": ["heat"]}) @@ -214,6 +220,7 @@ class HeatStacks(utils.HeatScenario): self.clients("heat").events.list(stack.id) @types.convert(template_path={"type": "file"}, files={"type": "file_dict"}) + @validation.validate_heat_template("template_path") @validation.required_services(consts.Service.HEAT) @validation.required_openstack(users=True) @scenario.configure(context={"cleanup": ["heat"]}) diff --git a/samples/tasks/scenarios/heat/create-and-delete-stack.json b/samples/tasks/scenarios/heat/create-and-delete-stack.json index 0887b180..3b47b767 100644 --- a/samples/tasks/scenarios/heat/create-and-delete-stack.json +++ b/samples/tasks/scenarios/heat/create-and-delete-stack.json @@ -2,7 +2,7 @@ "HeatStacks.create_and_delete_stack": [ { "args": { - "template_path": "templates/default.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/default.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-and-delete-stack.yaml b/samples/tasks/scenarios/heat/create-and-delete-stack.yaml index f6de6a9a..b665f668 100644 --- a/samples/tasks/scenarios/heat/create-and-delete-stack.yaml +++ b/samples/tasks/scenarios/heat/create-and-delete-stack.yaml @@ -2,7 +2,7 @@ HeatStacks.create_and_delete_stack: - args: - template_path: "templates/default.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/default.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-and-list-stack.json b/samples/tasks/scenarios/heat/create-and-list-stack.json index 961efd52..9208bd29 100644 --- a/samples/tasks/scenarios/heat/create-and-list-stack.json +++ b/samples/tasks/scenarios/heat/create-and-list-stack.json @@ -2,7 +2,7 @@ "HeatStacks.create_and_list_stack": [ { "args": { - "template_path": "templates/default.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/default.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-and-list-stack.yaml b/samples/tasks/scenarios/heat/create-and-list-stack.yaml index 61a1b345..7b037c74 100644 --- a/samples/tasks/scenarios/heat/create-and-list-stack.yaml +++ b/samples/tasks/scenarios/heat/create-and-list-stack.yaml @@ -2,7 +2,7 @@ HeatStacks.create_and_list_stack: - args: - template_path: "templates/default.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/default.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-check-delete-stack.json b/samples/tasks/scenarios/heat/create-check-delete-stack.json index 10997895..8a30e985 100644 --- a/samples/tasks/scenarios/heat/create-check-delete-stack.json +++ b/samples/tasks/scenarios/heat/create-check-delete-stack.json @@ -2,7 +2,7 @@ "HeatStacks.create_check_delete_stack": [ { "args": { - "template_path": "templates/random_strings.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-check-delete-stack.yaml b/samples/tasks/scenarios/heat/create-check-delete-stack.yaml index 41bfc382..2137b86a 100644 --- a/samples/tasks/scenarios/heat/create-check-delete-stack.yaml +++ b/samples/tasks/scenarios/heat/create-check-delete-stack.yaml @@ -2,7 +2,7 @@ HeatStacks.create_check_delete_stack: - args: - template_path: "templates/random_strings.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.json b/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.json index 21d0d672..7de47797 100644 --- a/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.json +++ b/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.json @@ -2,7 +2,7 @@ "HeatStacks.create_snapshot_restore_delete_stack": [ { "args": { - "template_path": "templates/random_strings.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.yaml b/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.yaml index f4e036f6..f1f03722 100644 --- a/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.yaml +++ b/samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.yaml @@ -2,7 +2,7 @@ HeatStacks.create_snapshot_restore_delete_stack: - args: - template_path: "templates/random_strings.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-stack-and-scale.json b/samples/tasks/scenarios/heat/create-stack-and-scale.json index 4afcf59f..adacafca 100644 --- a/samples/tasks/scenarios/heat/create-stack-and-scale.json +++ b/samples/tasks/scenarios/heat/create-stack-and-scale.json @@ -2,7 +2,7 @@ "HeatStacks.create_stack_and_scale": [ { "args": { - "template_path": "templates/autoscaling_group.yaml.template", + "template_path": "samples/tasks/scenarios/heat/templates/autoscaling_group.yaml.template", "output_key": "scaling_url", "delta": 1 }, diff --git a/samples/tasks/scenarios/heat/create-stack-and-scale.yaml b/samples/tasks/scenarios/heat/create-stack-and-scale.yaml index 33cc7fdf..52910fb7 100644 --- a/samples/tasks/scenarios/heat/create-stack-and-scale.yaml +++ b/samples/tasks/scenarios/heat/create-stack-and-scale.yaml @@ -2,7 +2,7 @@ HeatStacks.create_stack_and_scale: - args: - template_path: "templates/autoscaling_group.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/autoscaling_group.yaml.template" output_key: "scaling_url" delta: 1 runner: diff --git a/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.json b/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.json index e3170ed5..6caa729a 100644 --- a/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.json +++ b/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.json @@ -2,7 +2,7 @@ "HeatStacks.create_suspend_resume_delete_stack": [ { "args": { - "template_path": "templates/random_strings.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.yaml b/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.yaml index 32ee90ef..08f95ccb 100644 --- a/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.yaml +++ b/samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.yaml @@ -2,7 +2,7 @@ HeatStacks.create_suspend_resume_delete_stack: - args: - template_path: "templates/random_strings.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json index c36f759a..0ba63f0e 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json @@ -2,8 +2,8 @@ "HeatStacks.create_update_delete_stack": [ { "args": { - "template_path": "templates/random_strings.yaml.template", - "updated_template_path": "templates/updated_random_strings_add.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template", + "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml index 6a66d70a..9782ea96 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml @@ -2,8 +2,8 @@ HeatStacks.create_update_delete_stack: - args: - template_path: "templates/random_strings.yaml.template" - updated_template_path: "templates/updated_random_strings_add.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" + updated_template_path: "samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json index 5e460cc1..f50fbf93 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json @@ -2,8 +2,8 @@ "HeatStacks.create_update_delete_stack": [ { "args": { - "template_path": "templates/random_strings.yaml.template", - "updated_template_path": "templates/updated_random_strings_delete.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template", + "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml index 13e5432a..39a7bc52 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml @@ -2,8 +2,8 @@ HeatStacks.create_update_delete_stack: - args: - template_path: "templates/random_strings.yaml.template" - updated_template_path: "templates/updated_random_strings_delete.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" + updated_template_path: "samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-increase.json b/samples/tasks/scenarios/heat/create-update-delete-stack-increase.json index 027c2477..dbb17c60 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-increase.json +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-increase.json @@ -2,8 +2,8 @@ "HeatStacks.create_update_delete_stack": [ { "args": { - "template_path": "templates/resource_group.yaml.template", - "updated_template_path": "templates/updated_resource_group_increase.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/resource_group.yaml.template", + "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_resource_group_increase.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-increase.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-increase.yaml index 6835d526..efff3ba4 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-increase.yaml +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-increase.yaml @@ -2,8 +2,8 @@ HeatStacks.create_update_delete_stack: - args: - template_path: "templates/resource_group.yaml.template" - updated_template_path: "templates/updated_resource_group_increase.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/resource_group.yaml.template" + updated_template_path: "samples/tasks/scenarios/heat/templates/updated_resource_group_increase.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.json b/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.json index c4affc8d..6c721a25 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.json +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.json @@ -2,8 +2,8 @@ "HeatStacks.create_update_delete_stack": [ { "args": { - "template_path": "templates/autoscaling_policy.yaml.template", - "updated_template_path": "templates/updated_autoscaling_policy_inplace.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/autoscaling_policy.yaml.template", + "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_autoscaling_policy_inplace.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.yaml index bace8f6d..e67193a6 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.yaml +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-inplace.yaml @@ -2,8 +2,8 @@ HeatStacks.create_update_delete_stack: - args: - template_path: "templates/autoscaling_policy.yaml.template" - updated_template_path: "templates/updated_autoscaling_policy_inplace.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/autoscaling_policy.yaml.template" + updated_template_path: "samples/tasks/scenarios/heat/templates/updated_autoscaling_policy_inplace.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.json b/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.json index cf3a2df7..07e51121 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.json +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.json @@ -2,8 +2,8 @@ "HeatStacks.create_update_delete_stack": [ { "args": { - "template_path": "templates/resource_group.yaml.template", - "updated_template_path": "templates/updated_resource_group_reduce.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/resource_group.yaml.template", + "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_resource_group_reduce.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.yaml index 9a973333..3d274ed5 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.yaml +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-reduce.yaml @@ -2,8 +2,8 @@ HeatStacks.create_update_delete_stack: - args: - template_path: "templates/resource_group.yaml.template" - updated_template_path: "templates/updated_resource_group_reduce.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/resource_group.yaml.template" + updated_template_path: "samples/tasks/scenarios/heat/templates/updated_resource_group_reduce.yaml.template" runner: type: "constant" times: 10 diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-replace.json b/samples/tasks/scenarios/heat/create-update-delete-stack-replace.json index ffe59364..6522b6ea 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-replace.json +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-replace.json @@ -2,8 +2,8 @@ "HeatStacks.create_update_delete_stack": [ { "args": { - "template_path": "templates/random_strings.yaml.template", - "updated_template_path": "templates/updated_random_strings_replace.yaml.template" + "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template", + "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_random_strings_replace.yaml.template" }, "runner": { "type": "constant", diff --git a/samples/tasks/scenarios/heat/create-update-delete-stack-replace.yaml b/samples/tasks/scenarios/heat/create-update-delete-stack-replace.yaml index 4c60677e..820ce6e8 100644 --- a/samples/tasks/scenarios/heat/create-update-delete-stack-replace.yaml +++ b/samples/tasks/scenarios/heat/create-update-delete-stack-replace.yaml @@ -2,8 +2,8 @@ HeatStacks.create_update_delete_stack: - args: - template_path: "templates/random_strings.yaml.template" - updated_template_path: "templates/updated_random_strings_replace.yaml.template" + template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template" + updated_template_path: "samples/tasks/scenarios/heat/templates/updated_random_strings_replace.yaml.template" runner: type: "constant" times: 10