Wire in jinja templating for custom roles

This patch ports the custom roles code contained in the
proposed patch: I1851d1dca30bf2df1f0b99ec2a97c819d8420b71

This patch integrates an optional path to include jinja templating
if the jinja and roles files are present in the swift container
during the ProcessTemplatesAction.  This action is called before
any parameters management or deployment actions and should not require
any changes to the overall deployment plan management workflow.

If a roles_data.yaml file is found in the plan, we then iterate over
all files in the container, and render any foo.j2.yaml files with
the data from roles_data.yaml, then push the result as foo.yaml
back to the plan.

Co-Authored-By: Steven Hardy <shardy@redhat.com>
Change-Id: Iff88d1a74cf608885eed92077f85accabf1e672e
Partially-Implements: blueprint custom-roles

requirements.txt

@@ -14,3 +14,4 @@ python-ironicclient>=1.6.0 # Apache-2.0
 six>=1.9.0 # MIT
 mistral!=2015.1.0,>=2.0.0 # Apache-2.0
 python-ironic-inspector-client>=1.5.0 # Apache-2.0
+Jinja2>=2.8 # BSD License (3 clause)

tripleo_common/actions/templates.py

@@ -12,15 +12,19 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import jinja2
 import json
 import logging
 import os
 import requests
+import six
 import tempfile as tf
+import yaml
 
 from heatclient.common import template_utils
 from mistral import context
 from mistral.workflow import utils as mistral_workflow_utils
+from swiftclient import exceptions as swiftexceptions
 
 from tripleo_common.actions import base
 from tripleo_common import constants
@@ -38,9 +42,7 @@ def _create_temp_file(data):
 
 
 class UploadTemplatesAction(base.TripleOAction):
-    """Upload default heat templates for TripleO.
-
-    """
+    """Upload default heat templates for TripleO."""
     def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
         super(UploadTemplatesAction, self).__init__()
         self.container = container
@@ -56,11 +58,63 @@ class UploadTemplatesAction(base.TripleOAction):
 
 
 class ProcessTemplatesAction(base.TripleOAction):
+    """Process Templates and Environments
+
+    This method processes the templates and files in a given deployment
+    plan into a format that can be passed to Heat.
+    """
 
     def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
         super(ProcessTemplatesAction, self).__init__()
         self.container = container
 
+    def _process_custom_roles(self):
+        swift = self._get_object_client()
+        try:
+            j2_role_file = swift.get_object(
+                self.container, constants.OVERCLOUD_J2_ROLES_NAME)[1]
+            role_data = yaml.safe_load(j2_role_file)
+        except swiftexceptions.ClientException:
+            LOG.info("No %s file found, skipping jinja templating"
+                     % constants.OVERCLOUD_J2_ROLES_NAME)
+            return
+        try:
+            # Iterate over all files in the plan container
+            # we j2 render any with the .j2.yaml suffix
+            container_files = swift.get_container(self.container)
+        except swiftexceptions.ClientException as ex:
+            error_msg = ("Error listing contents of container %s : %s"
+                         % (self.container, six.text_type(ex)))
+            LOG.error(error_msg)
+            raise Exception(error_msg)
+
+        for f in [f.get('name') for f in container_files[1]]:
+            # check to see if the file is a *.j2.yaml
+            # if it is, get it and template for roles
+            if f.endswith('.j2.yaml'):
+                LOG.info("jinja2 rendering %s" % f)
+                j2_template = swift.get_object(self.container, f)[1]
+
+                try:
+                    # Render the j2 template
+                    template = jinja2.Environment().from_string(j2_template)
+                    r_template = template.render(roles=role_data)
+                except jinja2.exceptions.TemplateError as ex:
+                    error_msg = ("Error rendering template %s : %s"
+                                 % (f, six.text_type(ex)))
+                    LOG.error(error_msg)
+                    raise Exception(error_msg)
+                try:
+                    # write the template back to the plan container
+                    yaml_f = f.replace('.j2.yaml', '.yaml')
+                    swift.put_object(
+                        self.container, yaml_f, r_template)
+                except swiftexceptions.ClientException as ex:
+                    error_msg = ("Error storing file %s in container %s"
+                                 % (yaml_f, self.container))
+                    LOG.error(error_msg)
+                    raise Exception(error_msg)
+
     def run(self):
         error_text = None
         ctx = context.ctx()
@@ -69,11 +123,23 @@ class ProcessTemplatesAction(base.TripleOAction):
         try:
             mistral_environment = mistral.environments.get(self.container)
         except Exception as mistral_err:
-            error_text = mistral_err.message
+            error_text = six.text_type(mistral_err)
             LOG.exception(
                 "Error retrieving Mistral Environment: %s" % self.container)
             return mistral_workflow_utils.Result(error=error_text)
 
+        try:
+            # if the jinja overcloud template exists, process it and write it
+            # back to the swift container before continuing processing.  The
+            # method called below should handle the case where the files are
+            # not found in swift, but if they are found and an exception
+            # occurs during processing, that exception will cause the
+            # ProcessTemplatesAction to return an error result.
+            self._process_custom_roles()
+        except Exception as err:
+            LOG.exception("Error occurred while processing custom roles.")
+            return mistral_workflow_utils.Result(error=six.text_type(err))
+
         template_name = mistral_environment.variables.get('template')
         environments = mistral_environment.variables.get('environments')
         env_paths = []
@@ -122,7 +188,7 @@ class ProcessTemplatesAction(base.TripleOAction):
                     object_request=_object_request))
 
         except Exception as err:
-            error_text = str(err)
+            error_text = six.text_type(err)
             LOG.exception("Error occurred while processing plan files.")
         finally:
             # cleanup any local temp files

tripleo_common/constants.py

@@ -14,8 +14,15 @@
 # under the License.
 
 
-#: The name of the root template in a standard tripleo-heat-template layout.
+#: The names of the root template in a standard tripleo-heat-template layout.
 TEMPLATE_NAME = 'overcloud-without-mergepy.yaml'
+OVERCLOUD_YAML_NAME = "overcloud.yaml"
+
+#: The name of the overcloud root template in jinja2 format.
+OVERCLOUD_J2_NAME = "overcloud.j2.yaml"
+
+#: The name of custom roles data file used when rendering the jinja template.
+OVERCLOUD_J2_ROLES_NAME = "roles_data.yaml"
 
 #: The name of the type for resource groups.
 RESOURCE_GROUP_TYPE = 'OS::Heat::ResourceGroup'

tripleo_common/tests/actions/test_deployment.py

@@ -15,6 +15,7 @@
 import mock
 
 from mistral.workflow import utils as mistral_workflow_utils
+from swiftclient import exceptions as swiftexceptions
 
 from tripleo_common.actions import deployment
 from tripleo_common.tests import base
@@ -208,8 +209,12 @@ class DeployStackActionTest(base.TestCase):
                  mock_process_multiple_environments_and_files):
 
         mock_ctx.return_value = mock.MagicMock()
-        mock_get_object_client.return_value = mock.MagicMock(
-            url="http://test.com")
+        # setup swift
+        swift = mock.MagicMock(url="http://test.com")
+        swift.get_object.side_effect = swiftexceptions.ClientException(
+            'atest2')
+        mock_get_object_client.return_value = swift
+
         heat = mock.MagicMock()
         heat.stacks.get.return_value = None
         get_orchestration_client_mock.return_value = heat

tripleo_common/tests/actions/test_parameters.py

@@ -14,6 +14,8 @@
 # under the License.
 import mock
 
+from swiftclient import exceptions as swiftexceptions
+
 from tripleo_common.actions import parameters
 from tripleo_common import constants
 from tripleo_common.tests import base
@@ -36,8 +38,10 @@ class GetParametersActionTest(base.TestCase):
                  mock_process_multiple_environments_and_files):
 
         mock_ctx.return_value = mock.MagicMock()
-        mock_get_object_client.return_value = mock.MagicMock(
-            url="http://test.com")
+        swift = mock.MagicMock(url="http://test.com")
+        swift.get_object.side_effect = swiftexceptions.ClientException(
+            'atest2')
+        mock_get_object_client.return_value = swift
 
         mock_mistral = mock.MagicMock()
         mock_env = mock.MagicMock()

tripleo_common/tests/actions/test_templates.py

@@ -14,11 +14,48 @@
 # under the License.
 import mock
 
+from swiftclient import exceptions as swiftexceptions
+
 from tripleo_common.actions import templates
 from tripleo_common import constants
 from tripleo_common.tests import base
 
 
+JINJA_SNIPPET = """
+# Jinja loop for Role in role_data.yaml
+{% for role in roles %}
+  # Resources generated for {{role.name}} Role
+  {{role.name}}ServiceChain:
+    type: OS::TripleO::Services
+    properties:
+      Services:
+        get_param: {{role.name}}Services
+      ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
+      EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
+      DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
+{% endfor %}"""
+
+ROLE_DATA_YAML = """
+-
+  name: Controller
+"""
+
+
+EXPECTED_JINJA_RESULT = """
+# Jinja loop for Role in role_data.yaml
+
+  # Resources generated for Controller Role
+  ControllerServiceChain:
+    type: OS::TripleO::Services
+    properties:
+      Services:
+        get_param: ControllerServices
+      ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
+      EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
+      DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
+"""
+
+
 class UploadTemplatesActionTest(base.TestCase):
 
     @mock.patch('tempfile.NamedTemporaryFile')
@@ -54,8 +91,10 @@ class ProcessTemplatesActionTest(base.TestCase):
                  mock_process_multiple_environments_and_files):
 
         mock_ctx.return_value = mock.MagicMock()
-        mock_get_object_client.return_value = mock.MagicMock(
-            url="http://test.com")
+        swift = mock.MagicMock(url="http://test.com")
+        swift.get_object.side_effect = swiftexceptions.ClientException(
+            'atest2')
+        mock_get_object_client.return_value = swift
 
         mock_mistral = mock.MagicMock()
         mock_env = mock.MagicMock()
@@ -85,3 +124,50 @@ class ProcessTemplatesActionTest(base.TestCase):
                 'heat_template_version': '2016-04-30'
             }
         })
+
+    @mock.patch('tripleo_common.actions.base.TripleOAction._get_object_client')
+    @mock.patch('mistral.context.ctx')
+    def test_process_custom_roles(self, ctx_mock, get_obj_client_mock):
+
+        def return_multiple_files(*args):
+            if args[1] == constants.OVERCLOUD_J2_NAME:
+                return ['', JINJA_SNIPPET]
+            if args[1] == 'foo.j2.yaml':
+                return ['', JINJA_SNIPPET]
+            elif args[1] == constants.OVERCLOUD_J2_ROLES_NAME:
+                return ['', ROLE_DATA_YAML]
+
+        def return_container_files(*args):
+            return ('headers', [{'name': constants.OVERCLOUD_J2_NAME},
+                                {'name': 'foo.j2.yaml'},
+                                {'name': constants.OVERCLOUD_J2_ROLES_NAME}])
+
+        # setup swift
+        swift = mock.MagicMock()
+        swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
+        swift.get_container = mock.MagicMock(
+            side_effect=return_container_files)
+        get_obj_client_mock.return_value = swift
+
+        # Test
+        action = templates.ProcessTemplatesAction()
+        action._process_custom_roles()
+
+        get_object_mock_calls = [
+            mock.call('overcloud', constants.OVERCLOUD_J2_NAME),
+            mock.call('overcloud', constants.OVERCLOUD_J2_ROLES_NAME),
+            mock.call('overcloud', 'foo.j2.yaml'),
+        ]
+        swift.get_object.assert_has_calls(
+            get_object_mock_calls, any_order=True)
+
+        put_object_mock_calls = [
+            mock.call(constants.DEFAULT_CONTAINER_NAME,
+                      constants.OVERCLOUD_YAML_NAME,
+                      EXPECTED_JINJA_RESULT),
+            mock.call(constants.DEFAULT_CONTAINER_NAME,
+                      'foo.yaml',
+                      EXPECTED_JINJA_RESULT),
+        ]
+        swift.put_object.assert_has_calls(
+            put_object_mock_calls, any_order=True)

workbooks/plan_management.yaml

@@ -13,21 +13,32 @@ workflows:
     tasks:
       create_plan:
         action: tripleo.create_plan container=<% $.container %>
+        on-success: process_templates
+        on-error: create_plan_set_status_failed
+
+      process_templates:
+        action: tripleo.process_templates container=<% $.container %>
         on-success: set_status_success
-        on-error: set_status_failed
+        on-error: process_templates_set_status_failed
 
       set_status_success:
         on-success: notify_zaqar
         publish:
           status: SUCCESS
-          message: <% task(create_plan).result %>
+          message: <% task(process_templates).result %>
 
-      set_status_failed:
+      create_plan_set_status_failed:
         on-success: notify_zaqar
         publish:
           status: FAILED
           message: <% task(create_plan).result %>
 
+      process_templates_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(process_templates).result %>
+
       notify_zaqar:
         action: zaqar.queue_post
         input:
@@ -75,6 +86,11 @@ workflows:
 
       create_plan:
         action: tripleo.create_plan container=<% $.container %>
+        on-success: plan_process_templates
+        on-error: plan_set_status_failed
+
+      plan_process_templates:
+        action: tripleo.process_templates container=<% $.container %>
         on-success: plan_set_status_success
         on-error: plan_set_status_failed
 
@@ -90,6 +106,12 @@ workflows:
           status: FAILED
           message: <% task(create_plan).result %>
 
+      process_templates_set_status_failed:
+        on-success: notify_zaqar
+        publish:
+          status: FAILED
+          message: <% task(plan_process_templates).result %>
+
       upload_set_status_failed:
         on-success: notify_zaqar
         publish: