Parse and render yaml files in the proper order
This commit introduces a clear order of loading jvars: - passwords.yml - globals.yml - custom variables based on globals.yml - config/<service>/all.yml - config/<service/[...] It also introduces a mechanism of jinja2 templating yaml files by themselves, which means that one variable in yaml file can be reused in the next variables inside the same file. TrivialFix Change-Id: I1a7a61e90963d396702f5e0c14eb476d3ccd21bf
This commit is contained in:
parent
870cc8c88e
commit
36fcfb9df2
|
@ -10,17 +10,40 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import collections
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
|
||||
import jinja2
|
||||
from jinja2 import meta
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from kolla_mesos.common import type_utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Customize PyYAML library to return the OrderedDict. That is needed, because
|
||||
# when iterating on dict, we reuse its previous values when processing the
|
||||
# next values and the order has to be preserved.
|
||||
|
||||
def ordered_dict_constructor(loader, node):
|
||||
"""OrderedDict constructor for PyYAML."""
|
||||
return collections.OrderedDict(loader.construct_pairs(node))
|
||||
|
||||
|
||||
def ordered_dict_representer(dumper, data):
|
||||
"""Representer for PyYAML which is able to work with OrderedDict."""
|
||||
return dumper.represent_dict(data.items())
|
||||
|
||||
|
||||
yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
|
||||
ordered_dict_constructor)
|
||||
yaml.add_representer(collections.OrderedDict, ordered_dict_representer)
|
||||
|
||||
|
||||
def jinja_render(fullpath, global_config, extra=None):
|
||||
variables = global_config
|
||||
if extra:
|
||||
|
@ -51,3 +74,34 @@ def jinja_find_required_variables(fullpath):
|
|||
os.path.basename(fullpath))[0]
|
||||
parsed_content = myenv.parse(template_source)
|
||||
return meta.find_undeclared_variables(parsed_content)
|
||||
|
||||
|
||||
def dict_jinja_render(raw_dict, jvars):
|
||||
"""Renders dict with jinja2 using provided variables and itself.
|
||||
|
||||
By using itself, we mean reusing the previous values from dict for the
|
||||
potential render of the next value in dict.
|
||||
"""
|
||||
rendered_dict = {}
|
||||
for key, value in raw_dict.items():
|
||||
if isinstance(value, six.string_types):
|
||||
value = jinja_render_str(value, jvars)
|
||||
elif isinstance(value, dict):
|
||||
value = dict_jinja_render(value, jvars)
|
||||
rendered_dict[key] = value
|
||||
jvars[key] = value
|
||||
return rendered_dict
|
||||
|
||||
|
||||
def yaml_jinja_render(filename, variables):
|
||||
"""Parses YAML file and templates it with jinja2.
|
||||
|
||||
1. YAML file is rendered by jinja2 based on the provided variables.
|
||||
2. Rendered file is parsed.
|
||||
3. The every element dictionary being a result of parsing is rendered again
|
||||
with itself.
|
||||
"""
|
||||
jvars = copy.deepcopy(variables)
|
||||
with open(filename, 'r') as yaml_file:
|
||||
raw_dict = yaml.load(yaml_file)
|
||||
return dict_jinja_render(raw_dict, jvars)
|
||||
|
|
|
@ -468,34 +468,29 @@ def _load_variables_from_zk(zk):
|
|||
def _load_variables_from_file(service_dir, project_name):
|
||||
config_dir = os.path.join(service_dir, '..', 'config')
|
||||
with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
|
||||
global_vars = yaml.load(gf)
|
||||
jvars = yaml.load(gf)
|
||||
with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
|
||||
global_vars.update(yaml.load(gf))
|
||||
jvars.update(yaml.load(gf))
|
||||
# Apply the basic variables that aren't defined in any config file.
|
||||
jvars.update({
|
||||
'deployment_id': CONF.kolla.deployment_id,
|
||||
'node_config_directory': '',
|
||||
'timestamp': str(time.time())
|
||||
})
|
||||
# Apply the dynamic variables after reading raw variables.
|
||||
config.apply_deployment_vars(jvars)
|
||||
config.get_marathon_framework(jvars)
|
||||
# all.yml file uses some its variables to template itself by jinja2,
|
||||
# so its raw content is used to template the file
|
||||
all_yml_name = os.path.join(config_dir, 'all.yml')
|
||||
with open(all_yml_name) as af:
|
||||
raw_vars = yaml.load(af)
|
||||
raw_vars.update(global_vars)
|
||||
jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars))
|
||||
jvars.update(global_vars)
|
||||
jvars.update(jinja_utils.yaml_jinja_render(all_yml_name, jvars))
|
||||
|
||||
proj_yml_name = os.path.join(config_dir, project_name,
|
||||
'defaults', 'main.yml')
|
||||
if os.path.exists(proj_yml_name):
|
||||
proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
|
||||
jvars))
|
||||
jvars.update(proj_vars)
|
||||
jvars.update(jinja_utils.yaml_jinja_render(proj_yml_name, jvars))
|
||||
else:
|
||||
LOG.warning('Path missing %s' % proj_yml_name)
|
||||
# Add deployment_id
|
||||
jvars.update({'deployment_id': CONF.kolla.deployment_id})
|
||||
# override node_config_directory to empty
|
||||
jvars.update({'node_config_directory': ''})
|
||||
# Add timestamp
|
||||
jvars.update({'timestamp': str(time.time())})
|
||||
config.apply_deployment_vars(jvars)
|
||||
config.get_marathon_framework(jvars)
|
||||
return jvars
|
||||
|
||||
|
||||
|
|
|
@ -0,0 +1,29 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import collections
|
||||
|
||||
from kolla_mesos.common import jinja_utils
|
||||
from kolla_mesos.tests import base
|
||||
|
||||
|
||||
class TestJinjaUtils(base.BaseTestCase):
|
||||
|
||||
def test_dict_jinja_render(self):
|
||||
raw_dict = collections.OrderedDict([
|
||||
('first_key', '{{ test_var }}_test',),
|
||||
('second_key', '{{ first_key }}_test'),
|
||||
])
|
||||
jvars = {'test_var': 'test'}
|
||||
rendered_dict = jinja_utils.dict_jinja_render(raw_dict, jvars)
|
||||
self.assertEqual(rendered_dict['first_key'], 'test_test')
|
||||
self.assertEqual(rendered_dict['second_key'], 'test_test_test')
|
Loading…
Reference in New Issue