Parse and render yaml files in the proper order

This commit introduces a clear order of loading jvars:
- passwords.yml
- globals.yml
- custom variables based on globals.yml
- config/<service>/all.yml
- config/<service/[...]

It also introduces a mechanism of jinja2 templating yaml
files by themselves, which means that one variable in yaml
file can be reused in the next variables inside the same file.

TrivialFix

Change-Id: I1a7a61e90963d396702f5e0c14eb476d3ccd21bf

kolla_mesos/common/jinja_utils.py

@@ -10,17 +10,40 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
+import copy
 import logging
 import os
 
 import jinja2
 from jinja2 import meta
+import six
+import yaml
 
 from kolla_mesos.common import type_utils
 
 LOG = logging.getLogger(__name__)
 
 
+# Customize PyYAML library to return the OrderedDict. That is needed, because
+# when iterating on dict, we reuse its previous values when processing the
+# next values and the order has to be preserved.
+
+def ordered_dict_constructor(loader, node):
+    """OrderedDict constructor for PyYAML."""
+    return collections.OrderedDict(loader.construct_pairs(node))
+
+
+def ordered_dict_representer(dumper, data):
+    """Representer for PyYAML which is able to work with OrderedDict."""
+    return dumper.represent_dict(data.items())
+
+
+yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                     ordered_dict_constructor)
+yaml.add_representer(collections.OrderedDict, ordered_dict_representer)
+
+
 def jinja_render(fullpath, global_config, extra=None):
     variables = global_config
     if extra:
@@ -51,3 +74,34 @@ def jinja_find_required_variables(fullpath):
                                               os.path.basename(fullpath))[0]
     parsed_content = myenv.parse(template_source)
     return meta.find_undeclared_variables(parsed_content)
+
+
+def dict_jinja_render(raw_dict, jvars):
+    """Renders dict with jinja2 using provided variables and itself.
+
+    By using itself, we mean reusing the previous values from dict for the
+    potential render of the next value in dict.
+    """
+    rendered_dict = {}
+    for key, value in raw_dict.items():
+        if isinstance(value, six.string_types):
+            value = jinja_render_str(value, jvars)
+        elif isinstance(value, dict):
+            value = dict_jinja_render(value, jvars)
+        rendered_dict[key] = value
+        jvars[key] = value
+    return rendered_dict
+
+
+def yaml_jinja_render(filename, variables):
+    """Parses YAML file and templates it with jinja2.
+
+    1. YAML file is rendered by jinja2 based on the provided variables.
+    2. Rendered file is parsed.
+    3. The every element dictionary being a result of parsing is rendered again
+       with itself.
+    """
+    jvars = copy.deepcopy(variables)
+    with open(filename, 'r') as yaml_file:
+        raw_dict = yaml.load(yaml_file)
+    return dict_jinja_render(raw_dict, jvars)

kolla_mesos/service.py

@@ -10,6 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
 import functools
 import itertools
 import json
@@ -468,34 +469,33 @@ def _load_variables_from_zk(zk):
 def _load_variables_from_file(service_dir, project_name):
     config_dir = os.path.join(service_dir, '..', 'config')
     with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
-        global_vars = yaml.load(gf)
+        jvars = yaml.load(gf)
     with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
-        global_vars.update(yaml.load(gf))
+        jvars.update(yaml.load(gf))
+    global_vars = copy.deepcopy(jvars)
+    # Apply the basic variables that aren't defined in any config file.
+    jvars.update({
+        'deployment_id': CONF.kolla.deployment_id,
+        'node_config_directory': '',
+        'timestamp': str(time.time())
+    })
+    # Apply the dynamic variables after reading raw variables.
+    config.apply_deployment_vars(jvars)
+    config.get_marathon_framework(jvars)
     # all.yml file uses some its variables to template itself by jinja2,
     # so its raw content is used to template the file
     all_yml_name = os.path.join(config_dir, 'all.yml')
-    with open(all_yml_name) as af:
-        raw_vars = yaml.load(af)
-    raw_vars.update(global_vars)
-    jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars))
-    jvars.update(global_vars)
+    jvars.update(jinja_utils.yaml_jinja_render(all_yml_name, jvars))
 
     proj_yml_name = os.path.join(config_dir, project_name,
                                  'defaults', 'main.yml')
     if os.path.exists(proj_yml_name):
-        proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
-                                                       jvars))
-        jvars.update(proj_vars)
+        jvars.update(jinja_utils.yaml_jinja_render(proj_yml_name, jvars))
     else:
         LOG.warning('Path missing %s' % proj_yml_name)
-    # Add deployment_id
-    jvars.update({'deployment_id': CONF.kolla.deployment_id})
-    # override node_config_directory to empty
-    jvars.update({'node_config_directory': ''})
-    # Add timestamp
-    jvars.update({'timestamp': str(time.time())})
-    config.apply_deployment_vars(jvars)
-    config.get_marathon_framework(jvars)
+
+    # Re-apply the global variables to ensure they weren't overriden.
+    jvars.update(global_vars)
     return jvars
 
 

kolla_mesos/tests/common/test_jinja_utils.py

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+from kolla_mesos.common import jinja_utils
+from kolla_mesos.tests import base
+
+
+class TestJinjaUtils(base.BaseTestCase):
+
+    def test_dict_jinja_render(self):
+        raw_dict = collections.OrderedDict([
+            ('first_key', '{{ test_var }}_test',),
+            ('second_key', '{{ first_key }}_test'),
+        ])
+        jvars = {'test_var': 'test'}
+        rendered_dict = jinja_utils.dict_jinja_render(raw_dict, jvars)
+        self.assertEqual(rendered_dict['first_key'], 'test_test')
+        self.assertEqual(rendered_dict['second_key'], 'test_test_test')