Merge "[Bug1535948] Add Heat template validator"

rally-jobs/rally-mos.yaml

@@ -129,6 +129,8 @@
 
   HeatStacks.create_and_list_stack:
     -
+      args:
+        template_path: "~/.rally/extra/default.yaml.template"
       runner:
         type: "constant"
         times: 1
@@ -137,9 +139,14 @@
         users:
           tenants: 1
           users_per_tenant: 1
+      sla:
+        failure_rate:
+          max: 0
 
   HeatStacks.create_and_delete_stack:
     -
+      args:
+        template_path: "~/.rally/extra/default.yaml.template"
       runner:
         type: "constant"
         times: 1
@@ -148,6 +155,9 @@
         users:
           tenants: 1
           users_per_tenant: 1
+      sla:
+        failure_rate:
+          max: 0
 
   Authenticate.keystone:
     -
@@ -534,6 +544,10 @@
           nova:
             security_groups: -1
             security_group_rules: -1
+      sla:
+        failure_rate:
+          max: 0
+
 
 
   NeutronNetworks.create_and_list_networks:

rally/plugins/openstack/scenarios/heat/stacks.py

@@ -25,6 +25,7 @@ class HeatStacks(utils.HeatScenario):
     """Benchmark scenarios for Heat stacks."""
 
     @types.convert(template_path={"type": "file"}, files={"type": "file_dict"})
+    @validation.validate_heat_template("template_path")
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @scenario.configure(context={"cleanup": ["heat"]})
@@ -56,6 +57,7 @@ class HeatStacks(utils.HeatScenario):
                 self.clients("heat").resources.list(stack.id)
 
     @types.convert(template_path={"type": "file"}, files={"type": "file_dict"})
+    @validation.validate_heat_template("template_path")
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @scenario.configure(context={"cleanup": ["heat"]})
@@ -77,6 +79,7 @@ class HeatStacks(utils.HeatScenario):
         self._delete_stack(stack)
 
     @types.convert(template_path={"type": "file"}, files={"type": "file_dict"})
+    @validation.validate_heat_template("template_path")
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @scenario.configure(context={"cleanup": ["heat"]})
@@ -104,6 +107,7 @@ class HeatStacks(utils.HeatScenario):
                    updated_template_path={"type": "file"},
                    files={"type": "file_dict"},
                    updated_files={"type": "file_dict"})
+    @validation.validate_heat_template("template_path")
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @scenario.configure(context={"cleanup": ["heat"]})
@@ -139,6 +143,7 @@ class HeatStacks(utils.HeatScenario):
         self._delete_stack(stack)
 
     @types.convert(template_path={"type": "file"}, files={"type": "file_dict"})
+    @validation.validate_heat_template("template_path")
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @scenario.configure(context={"cleanup": ["heat"]})
@@ -176,6 +181,7 @@ class HeatStacks(utils.HeatScenario):
         self._scale_stack(stack, output_key, delta)
 
     @types.convert(template_path={"type": "file"}, files={"type": "file_dict"})
+    @validation.validate_heat_template("template_path")
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @scenario.configure(context={"cleanup": ["heat"]})
@@ -214,6 +220,7 @@ class HeatStacks(utils.HeatScenario):
                 self.clients("heat").events.list(stack.id)
 
     @types.convert(template_path={"type": "file"}, files={"type": "file_dict"})
+    @validation.validate_heat_template("template_path")
     @validation.required_services(consts.Service.HEAT)
     @validation.required_openstack(users=True)
     @scenario.configure(context={"cleanup": ["heat"]})

rally/task/validation.py

@@ -649,3 +649,37 @@ def restricted_parameters(config, clients, deployment, param_names,
                % {"params": ", ".join(restricted_params),
                   "a_dict": subdict if subdict else "args"})
         return ValidationResult(False, msg)
+
+
+@validator
+def validate_heat_template(config, clients, deployment, *param_names):
+    """Validates heat template.
+
+    :param param_names: list of parameters to be validated.
+    """
+    if param_names is None:
+        return ValidationResult(False, _(
+            "validate_heat_template validator accepts non empty arguments "
+            "in form of `validate_heat_template(\"foo\", \"bar\")`"))
+    for param_name in param_names:
+        template_path = config.get("args", {}).get(param_name)
+        if not template_path:
+            return ValidationResult(False, _(
+                "Path to heat template is not specified. Its needed for "
+                "heat template validation. Please check the content of `%s` "
+                "scenario argument.") % param_name)
+        template_path = os.path.expanduser(template_path)
+        if not os.path.exists(template_path):
+            return ValidationResult(False, _("No file found by the given path "
+                                             "%s") % template_path)
+        with open(template_path, "r") as f:
+            try:
+                clients.heat().stacks.validate(template=f.read())
+            except Exception as e:
+                dct = {
+                    "path": template_path,
+                    "msg": str(e),
+                }
+                msg = (_("Heat template validation failed on %(path)s. "
+                         "Original error message: %(msg)s.") % dct)
+                return ValidationResult(False, msg)

samples/tasks/scenarios/heat/create-and-delete-stack.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_and_delete_stack": [
         {
             "args": {
-               "template_path": "templates/default.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/default.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-and-delete-stack.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_and_delete_stack:
     -
       args:
-        template_path: "templates/default.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/default.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-and-delete-stack_resource_group.json

@@ -2,11 +2,11 @@
     "HeatStacks.create_and_delete_stack": [
         {
             "args": {
-               "template_path": "templates/resource_group_server_with_volume.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group_server_with_volume.yaml.template",
                "parameters": {
                    "num_instances": 2
                },
-               "files": ["templates/server_with_volume.yaml.template"]
+               "files": ["samples/tasks/scenarios/heat/templates/server_with_volume.yaml.template"]
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-and-delete-stack_resource_group.yaml

@@ -2,10 +2,10 @@
   HeatStacks.create_and_delete_stack:
     -
       args:
-        template_path: "templates/resource_group_server_with_volume.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group_server_with_volume.yaml.template"
         parameters:
            num_instances: 2
-        files: ["templates/server_with_volume.yaml.template"]
+        files: ["samples/tasks/scenarios/heat/templates/server_with_volume.yaml.template"]
       runner:
         type: "constant"
         times: 3

samples/tasks/scenarios/heat/create-and-delete-stack_with_delay.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_and_delete_stack": [
         {
             "args": {
-               "template_path": "templates/resource_group_with_constraint.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group_with_constraint.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-and-delete-stack_with_delay.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_and_delete_stack:
     -
       args:
-        template_path: "templates/resource_group_with_constraint.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group_with_constraint.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-and-delete-stack_with_neutron.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_and_delete_stack": [
         {
             "args": {
-               "template_path": "templates/server_with_ports.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/server_with_ports.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-and-delete-stack_with_neutron.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_and_delete_stack:
     -
       args:
-        template_path: "templates/server_with_ports.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/server_with_ports.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-and-delete-stack_with_volume.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_and_delete_stack": [
         {
             "args": {
-               "template_path": "templates/server_with_volume.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/server_with_volume.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-and-delete-stack_with_volume.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_and_delete_stack:
     -
       args:
-        template_path: "templates/server_with_volume.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/server_with_volume.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-and-list-stack.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_and_list_stack": [
         {
             "args": {
-               "template_path": "templates/default.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/default.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-and-list-stack.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_and_list_stack:
     -
       args:
-        template_path: "templates/default.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/default.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-check-delete-stack.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_check_delete_stack": [
         {
             "args": {
-              "template_path": "templates/random_strings.yaml.template"
+              "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-check-delete-stack.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_check_delete_stack:
     -
       args:
-        template_path: "templates/random_strings.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_snapshot_restore_delete_stack": [
         {
             "args": {
-               "template_path": "templates/random_strings.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-snapshot-restore-delete-stack.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_snapshot_restore_delete_stack:
     -
       args:
-        template_path: "templates/random_strings.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-stack-and-list-output_resource_group.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_stack_and_list_output": [
         {
             "args": {
-               "template_path": "templates/resource_group_with_outputs.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template"
             },
             "runner": {
                 "type": "constant",
@@ -21,7 +21,7 @@
     "HeatStacks.create_stack_and_list_output_via_API": [
         {
             "args": {
-               "template_path": "templates/resource_group_with_outputs.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-stack-and-list-output_resource_group.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_stack_and_list_output:
     -
       args:
-        template_path: "templates/resource_group_with_outputs.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template"
       runner:
         type: "constant"
         times: 5
@@ -15,7 +15,7 @@
   HeatStacks.create_stack_and_list_output_via_API:
     -
       args:
-        template_path: "templates/resource_group_with_outputs.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template"
       runner:
         type: "constant"
         times: 5

samples/tasks/scenarios/heat/create-stack-and-scale.json

@@ -2,7 +2,7 @@
   "HeatStacks.create_stack_and_scale": [
     {
       "args": {
-        "template_path": "templates/autoscaling_group.yaml.template",
+        "template_path": "samples/tasks/scenarios/heat/templates/autoscaling_group.yaml.template",
         "output_key": "scaling_url",
         "delta": 1
       },

samples/tasks/scenarios/heat/create-stack-and-scale.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_stack_and_scale:
     -
       args:
-        template_path: "templates/autoscaling_group.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/autoscaling_group.yaml.template"
         output_key: "scaling_url"
         delta: 1
       runner:

samples/tasks/scenarios/heat/create-stack-and-show-output_resource_group.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_stack_and_show_output": [
         {
             "args": {
-               "template_path": "templates/resource_group_with_outputs.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template",
                "output_key": "val1"
             },
             "runner": {
@@ -22,7 +22,7 @@
     "HeatStacks.create_stack_and_show_output_via_API": [
         {
             "args": {
-               "template_path": "templates/resource_group_with_outputs.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template",
                "output_key": "val1"
             },
             "runner": {

samples/tasks/scenarios/heat/create-stack-and-show-output_resource_group.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_stack_and_show_output:
     -
       args:
-        template_path: "templates/resource_group_with_outputs.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template"
         output_key: "val1"
       runner:
         type: "constant"
@@ -16,7 +16,7 @@
   HeatStacks.create_stack_and_show_output_via_API:
     -
       args:
-        template_path: "templates/resource_group_with_outputs.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group_with_outputs.yaml.template"
         output_key: "val1"
       runner:
         type: "constant"

samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.json

@@ -2,7 +2,7 @@
     "HeatStacks.create_suspend_resume_delete_stack": [
         {
             "args": {
-               "template_path": "templates/random_strings.yaml.template"
+               "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-suspend-resume-delete-stack.yaml

@@ -2,7 +2,7 @@
   HeatStacks.create_suspend_resume_delete_stack:
     -
       args:
-        template_path: "templates/random_strings.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-update-delete-stack-add-res.json

@@ -2,8 +2,8 @@
     "HeatStacks.create_update_delete_stack": [
         {
             "args": {
-               "template_path": "templates/random_strings.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template",
-               "updated_template_path": "templates/updated_random_strings_add.yaml.template"
+               "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-update-delete-stack-add-res.yaml

@@ -2,8 +2,8 @@
   HeatStacks.create_update_delete_stack:
     -
       args:
-        template_path: "templates/random_strings.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
-        updated_template_path: "templates/updated_random_strings_add.yaml.template"
+        updated_template_path: "samples/tasks/scenarios/heat/templates/updated_random_strings_add.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-update-delete-stack-del-res.json

@@ -2,8 +2,8 @@
     "HeatStacks.create_update_delete_stack": [
         {
             "args": {
-               "template_path": "templates/random_strings.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template",
-               "updated_template_path": "templates/updated_random_strings_delete.yaml.template"
+               "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-update-delete-stack-del-res.yaml

@@ -2,8 +2,8 @@
   HeatStacks.create_update_delete_stack:
     -
       args:
-        template_path: "templates/random_strings.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
-        updated_template_path: "templates/updated_random_strings_delete.yaml.template"
+        updated_template_path: "samples/tasks/scenarios/heat/templates/updated_random_strings_delete.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-update-delete-stack-increase.json

@@ -2,8 +2,8 @@
     "HeatStacks.create_update_delete_stack": [
         {
             "args": {
-               "template_path": "templates/resource_group.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group.yaml.template",
-               "updated_template_path": "templates/updated_resource_group_increase.yaml.template"
+               "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_resource_group_increase.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-update-delete-stack-increase.yaml

@@ -2,8 +2,8 @@
   HeatStacks.create_update_delete_stack:
     -
       args:
-        template_path: "templates/resource_group.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group.yaml.template"
-        updated_template_path: "templates/updated_resource_group_increase.yaml.template"
+        updated_template_path: "samples/tasks/scenarios/heat/templates/updated_resource_group_increase.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-update-delete-stack-inplace.json

@@ -2,8 +2,8 @@
     "HeatStacks.create_update_delete_stack": [
         {
             "args": {
-               "template_path": "templates/autoscaling_policy.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/autoscaling_policy.yaml.template",
-               "updated_template_path": "templates/updated_autoscaling_policy_inplace.yaml.template"
+               "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_autoscaling_policy_inplace.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-update-delete-stack-inplace.yaml

@@ -2,8 +2,8 @@
   HeatStacks.create_update_delete_stack:
     -
       args:
-        template_path: "templates/autoscaling_policy.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/autoscaling_policy.yaml.template"
-        updated_template_path: "templates/updated_autoscaling_policy_inplace.yaml.template"
+        updated_template_path: "samples/tasks/scenarios/heat/templates/updated_autoscaling_policy_inplace.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-update-delete-stack-reduce.json

@@ -2,8 +2,8 @@
     "HeatStacks.create_update_delete_stack": [
         {
             "args": {
-               "template_path": "templates/resource_group.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/resource_group.yaml.template",
-               "updated_template_path": "templates/updated_resource_group_reduce.yaml.template"
+               "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_resource_group_reduce.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-update-delete-stack-reduce.yaml

@@ -2,8 +2,8 @@
   HeatStacks.create_update_delete_stack:
     -
       args:
-        template_path: "templates/resource_group.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/resource_group.yaml.template"
-        updated_template_path: "templates/updated_resource_group_reduce.yaml.template"
+        updated_template_path: "samples/tasks/scenarios/heat/templates/updated_resource_group_reduce.yaml.template"
       runner:
         type: "constant"
         times: 10

samples/tasks/scenarios/heat/create-update-delete-stack-replace.json

@@ -2,8 +2,8 @@
     "HeatStacks.create_update_delete_stack": [
         {
             "args": {
-               "template_path": "templates/random_strings.yaml.template",
+               "template_path": "samples/tasks/scenarios/heat/templates/random_strings.yaml.template",
-               "updated_template_path": "templates/updated_random_strings_replace.yaml.template"
+               "updated_template_path": "samples/tasks/scenarios/heat/templates/updated_random_strings_replace.yaml.template"
             },
             "runner": {
                 "type": "constant",

samples/tasks/scenarios/heat/create-update-delete-stack-replace.yaml

@@ -2,8 +2,8 @@
   HeatStacks.create_update_delete_stack:
     -
       args:
-        template_path: "templates/random_strings.yaml.template"
+        template_path: "samples/tasks/scenarios/heat/templates/random_strings.yaml.template"
-        updated_template_path: "templates/updated_random_strings_replace.yaml.template"
+        updated_template_path: "samples/tasks/scenarios/heat/templates/updated_random_strings_replace.yaml.template"
       runner:
         type: "constant"
         times: 10

tests/unit/task/test_validation.py

@@ -843,3 +843,43 @@ class ValidatorsTestCase(test.TestCase):
             validation.restricted_parameters, "param_name")
         result = validator({"args": {}}, None, None)
         self.assertTrue(result.is_valid, result.msg)
+
+    @ddt.data(
+        {"exception_msg": "Heat template validation failed on fake_path1. "
+                          "Original error message: fake_msg."},
+        {"exception_msg": None}
+    )
+    @ddt.unpack
+    @mock.patch(MODULE + "os.path.exists", return_value=True)
+    @mock.patch(MODULE + "open", side_effect=mock.mock_open(), create=True)
+    def test_validate_heat_template(self, mock_open, mock_exists,
+                                    exception_msg):
+        validator = self._unwrap_validator(
+            validation.validate_heat_template, "template_path1",
+            "template_path2")
+        clients = mock.MagicMock()
+        mock_open().__enter__().read.side_effect = ["fake_template1",
+                                                    "fake_template2"]
+        heat_validator = mock.MagicMock()
+        if exception_msg:
+            heat_validator.side_effect = Exception("fake_msg")
+        clients.heat().stacks.validate = heat_validator
+        context = {"args": {"template_path1": "fake_path1",
+                            "template_path2": "fake_path2"}}
+        result = validator(context, clients, mock.MagicMock())
+
+        if not exception_msg:
+            heat_validator.assert_has_calls([
+                mock.call(template="fake_template1"),
+                mock.call(template="fake_template2")
+            ])
+            mock_open.assert_has_calls([
+                mock.call("fake_path1", "r"),
+                mock.call("fake_path2", "r")
+            ], any_order=True)
+            self.assertTrue(result.is_valid, result.msg)
+        else:
+            heat_validator.assert_called_once_with(template="fake_template1")
+            self.assertEqual("Heat template validation failed on fake_path1."
+                             " Original error message: fake_msg.", result.msg)
+            self.assertFalse(result.is_valid)