Merge "Modify roles to remove unused services"

tripleo_common/actions/templates.py

@@ -38,16 +38,10 @@ class J2SwiftLoader(jinja2.BaseLoader):
     only the absolute path relative to the container root is searched.
     """
 
-    def __init__(self, swift, container, searchpath=None):
+    def __init__(self, swift, container, searchpath):
         self.swift = swift
         self.container = container
-        if searchpath is not None:
+        self.searchpath = [searchpath]
-            if isinstance(searchpath, six.string_types):
-                self.searchpath = [searchpath]
-            else:
-                self.searchpath = list(searchpath)
-        else:
-            self.searchpath = []
         # Always search the absolute path from the root of the swift container
         if '' not in self.searchpath:
             self.searchpath.append('')
@@ -98,15 +92,9 @@ class ProcessTemplatesAction(base.TripleOAction):
     def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
         super(ProcessTemplatesAction, self).__init__()
         self.container = container
+        self.check_none = True
 
-    def _j2_render_and_put(self,
+    def _j2_render_and_put(self, j2_template, j2_data, yaml_f, swift):
-                           j2_template,
-                           j2_data,
-                           outfile_name=None,
-                           context=None):
-        swift = self.get_object_client(context)
-        yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
-
         # Search for templates relative to the current template path first
         template_base = os.path.dirname(yaml_f)
         j2_loader = J2SwiftLoader(swift, self.container, template_base)
@@ -286,7 +274,7 @@ class ProcessTemplatesAction(base.TripleOAction):
                             self._j2_render_and_put(j2_template,
                                                     j2_data,
                                                     out_f_path,
-                                                    context=context)
+                                                    swift)
                         else:
                             # Backwards compatibility with templates
                             # that specify {{role}} vs {{role.name}}
@@ -297,12 +285,12 @@ class ProcessTemplatesAction(base.TripleOAction):
                             self._j2_render_and_put(j2_template,
                                                     j2_data,
                                                     out_f_path,
-                                                    context=context)
+                                                    swift)
                     else:
                         LOG.info("Skipping rendering of %s, defined in %s" %
                                  (out_f_path, j2_excl_data))
 
-            elif (f.endswith('.network.j2.yaml')):
+            elif f.endswith('.network.j2.yaml'):
                 LOG.info("jinja2 rendering network template %s" % f)
                 j2_template = swiftutils.get_object_string(swift,
                                                            self.container,
@@ -324,7 +312,7 @@ class ProcessTemplatesAction(base.TripleOAction):
                         self._j2_render_and_put(j2_template,
                                                 j2_data,
                                                 out_f_path,
-                                                context=context)
+                                                swift)
                     else:
                         LOG.info("Skipping rendering of %s, defined in %s" %
                                  (out_f_path, j2_excl_data))
@@ -339,7 +327,8 @@ class ProcessTemplatesAction(base.TripleOAction):
                 self._j2_render_and_put(j2_template,
                                         j2_data,
                                         out_f,
-                                        context=context)
+                                        swift)
+        return role_data
 
     def run(self, context):
         error_text = None
@@ -361,7 +350,7 @@ class ProcessTemplatesAction(base.TripleOAction):
             # not found in swift, but if they are found and an exception
             # occurs during processing, that exception will cause the
             # ProcessTemplatesAction to return an error result.
-            self._process_custom_roles(context)
+            role_data = self._process_custom_roles(context)
         except Exception as err:
             LOG.exception("Error occurred while processing custom roles.")
             return actions.Result(error=six.text_type(err))
@@ -395,6 +384,26 @@ class ProcessTemplatesAction(base.TripleOAction):
         if error_text:
             return actions.Result(error=error_text)
 
+        if self.check_none and role_data:
+            to_remove = set()
+            for key, value in env.get('resource_registry', {}).items():
+                if (key.startswith('OS::TripleO::Services::') and
+                        value == 'OS::Heat::None'):
+                    to_remove.add(key)
+            if to_remove:
+                for role in role_data:
+                    for service in to_remove:
+                        try:
+                            role.get('ServicesDefault', []).remove(service)
+                        except ValueError:
+                            pass
+                LOG.info('Removing unused services, updating roles')
+                swift.put_object(
+                    self.container, constants.OVERCLOUD_J2_ROLES_NAME,
+                    yaml.safe_dump(role_data))
+                self.check_none = False
+                return ProcessTemplatesAction.run(self, context)
+
         files = dict(list(template_files.items()) + list(env_files.items()))
 
         return {

tripleo_common/tests/actions/test_templates.py

@@ -165,7 +165,7 @@ class J2SwiftLoaderTest(base.TestCase):
         return swift
 
     def test_include_absolute_path(self):
-        j2_loader = templates.J2SwiftLoader(self._setup_swift(), None)
+        j2_loader = templates.J2SwiftLoader(self._setup_swift(), None, '')
         template = jinja2.Environment(loader=j2_loader).from_string(
             r'''
             Included this:
@@ -193,7 +193,7 @@ class J2SwiftLoaderTest(base.TestCase):
             ''')
 
     def test_include_not_found(self):
-        j2_loader = templates.J2SwiftLoader(self._setup_swift(), None)
+        j2_loader = templates.J2SwiftLoader(self._setup_swift(), None, '')
         template = jinja2.Environment(loader=j2_loader).from_string(
             r'''
             Included this:
@@ -204,7 +204,7 @@ class J2SwiftLoaderTest(base.TestCase):
             template.render)
 
     def test_include_invalid_path(self):
-        j2_loader = templates.J2SwiftLoader(self._setup_swift(), 'bar')
+        j2_loader = templates.J2SwiftLoader(self._setup_swift(), 'bar', '')
         template = jinja2.Environment(loader=j2_loader).from_string(
             r'''
             Included this:
@@ -409,13 +409,12 @@ class ProcessTemplatesActionTest(base.TestCase):
         swift.get_object = mock.MagicMock()
         swift.get_container = mock.MagicMock()
         get_obj_client_mock.return_value = swift
-        mock_ctx = mock.MagicMock()
 
         # Test
         action = templates.ProcessTemplatesAction()
         action._j2_render_and_put(JINJA_SNIPPET_CONFIG,
                                   {'role': 'CustomRole'},
-                                  'customrole-config.yaml', context=mock_ctx)
+                                  'customrole-config.yaml', swift)
 
         action_result = swift.put_object._mock_mock_calls[0]
 
@@ -437,14 +436,13 @@ class ProcessTemplatesActionTest(base.TestCase):
         swift.get_container = mock.MagicMock(
             side_effect=return_container_files)
         get_obj_client_mock.return_value = swift
-        mock_ctx = mock.MagicMock()
 
         # Test
         action = templates.ProcessTemplatesAction()
         action._j2_render_and_put(r"{% include 'foo.yaml' %}",
                                   {'role': 'CustomRole'},
                                   'customrole-config.yaml',
-                                  context=mock_ctx)
+                                  swift)
 
         action_result = swift.put_object._mock_mock_calls[0]
 
@@ -468,14 +466,13 @@ class ProcessTemplatesActionTest(base.TestCase):
         swift.get_container = mock.MagicMock(
             side_effect=return_container_files)
         get_obj_client_mock.return_value = swift
-        mock_ctx = mock.MagicMock()
 
         # Test
         action = templates.ProcessTemplatesAction()
         action._j2_render_and_put(r"{% include 'foo.yaml' %}",
                                   {'role': 'CustomRole'},
                                   'bar/customrole-config.yaml',
-                                  context=mock_ctx)
+                                  swift)
 
         action_result = swift.put_object._mock_mock_calls[0]