Parse and render yaml files in the proper order
This commit introduces a clear order of loading jvars: - passwords.yml - globals.yml - custom variables based on globals.yml - config/<service>/all.yml - config/<service/[...] It also introduces a mechanism of jinja2 templating yaml files by themselves, which means that one variable in yaml file can be reused in the next variables inside the same file. TrivialFix Change-Id: I1a7a61e90963d396702f5e0c14eb476d3ccd21bf
This commit is contained in:
parent
5666ef4ff5
commit
ea39e9e5ed
|
@ -126,6 +126,16 @@ openstack_region_name: "RegionOne"
|
||||||
# Valid options are [ novnc, spice ]
|
# Valid options are [ novnc, spice ]
|
||||||
nova_console: "novnc"
|
nova_console: "novnc"
|
||||||
|
|
||||||
|
|
||||||
|
####################
|
||||||
|
# Constraints
|
||||||
|
####################
|
||||||
|
controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]'
|
||||||
|
compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]'
|
||||||
|
controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]'
|
||||||
|
storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]'
|
||||||
|
|
||||||
|
|
||||||
####################
|
####################
|
||||||
# Mesos-dns hosts
|
# Mesos-dns hosts
|
||||||
####################
|
####################
|
||||||
|
|
|
@ -42,14 +42,6 @@ controller_nodes: "1"
|
||||||
compute_nodes: "1"
|
compute_nodes: "1"
|
||||||
storage_nodes: "1"
|
storage_nodes: "1"
|
||||||
|
|
||||||
####################
|
|
||||||
# Constraints
|
|
||||||
####################
|
|
||||||
controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]'
|
|
||||||
compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]'
|
|
||||||
controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]'
|
|
||||||
storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]'
|
|
||||||
|
|
||||||
####################
|
####################
|
||||||
# OpenStack options
|
# OpenStack options
|
||||||
####################
|
####################
|
||||||
|
|
|
@ -10,17 +10,39 @@
|
||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
import collections
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import jinja2
|
import jinja2
|
||||||
from jinja2 import meta
|
from jinja2 import meta
|
||||||
|
import six
|
||||||
|
import yaml
|
||||||
|
|
||||||
from kolla_mesos.common import type_utils
|
from kolla_mesos.common import type_utils
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# Customize PyYAML library to return the OrderedDict. That is needed, because
|
||||||
|
# when iterating on dict, we reuse its previous values when processing the
|
||||||
|
# next values and the order has to be preserved.
|
||||||
|
|
||||||
|
def ordered_dict_constructor(loader, node):
|
||||||
|
"""OrderedDict constructor for PyYAML."""
|
||||||
|
return collections.OrderedDict(loader.construct_pairs(node))
|
||||||
|
|
||||||
|
|
||||||
|
def ordered_dict_representer(dumper, data):
|
||||||
|
"""Representer for PyYAML which is able to work with OrderedDict."""
|
||||||
|
return dumper.represent_dict(data.items())
|
||||||
|
|
||||||
|
|
||||||
|
yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
|
||||||
|
ordered_dict_constructor)
|
||||||
|
yaml.add_representer(collections.OrderedDict, ordered_dict_representer)
|
||||||
|
|
||||||
|
|
||||||
def jinja_render(fullpath, global_config, extra=None):
|
def jinja_render(fullpath, global_config, extra=None):
|
||||||
variables = global_config
|
variables = global_config
|
||||||
if extra:
|
if extra:
|
||||||
|
@ -51,3 +73,30 @@ def jinja_find_required_variables(fullpath):
|
||||||
os.path.basename(fullpath))[0]
|
os.path.basename(fullpath))[0]
|
||||||
parsed_content = myenv.parse(template_source)
|
parsed_content = myenv.parse(template_source)
|
||||||
return meta.find_undeclared_variables(parsed_content)
|
return meta.find_undeclared_variables(parsed_content)
|
||||||
|
|
||||||
|
|
||||||
|
def dict_jinja_render(raw_dict, jvars):
|
||||||
|
"""Renders dict with jinja2 using provided variables and itself.
|
||||||
|
|
||||||
|
By using itself, we mean reusing the previous values from dict for the
|
||||||
|
potential render of the next value in dict.
|
||||||
|
"""
|
||||||
|
for key, value in raw_dict.items():
|
||||||
|
if isinstance(value, six.string_types):
|
||||||
|
value = jinja_render_str(value, jvars)
|
||||||
|
elif isinstance(value, dict):
|
||||||
|
value = dict_jinja_render(value, jvars)
|
||||||
|
jvars[key] = value
|
||||||
|
|
||||||
|
|
||||||
|
def yaml_jinja_render(filename, jvars):
|
||||||
|
"""Parses YAML file and templates it with jinja2.
|
||||||
|
|
||||||
|
1. YAML file is rendered by jinja2 based on the provided variables.
|
||||||
|
2. Rendered file is parsed.
|
||||||
|
3. The every element dictionary being a result of parsing is rendered again
|
||||||
|
with itself.
|
||||||
|
"""
|
||||||
|
with open(filename, 'r') as yaml_file:
|
||||||
|
raw_dict = yaml.load(yaml_file)
|
||||||
|
dict_jinja_render(raw_dict, jvars)
|
||||||
|
|
|
@ -155,13 +155,13 @@ def apply_deployment_vars(jvars):
|
||||||
'controller_compute_constraints':
|
'controller_compute_constraints':
|
||||||
controller_compute_constraints,
|
controller_compute_constraints,
|
||||||
'storage_constraints': storage_constraints
|
'storage_constraints': storage_constraints
|
||||||
})
|
}, force=True)
|
||||||
jvars.update({
|
jvars.update({
|
||||||
'controller_nodes': str(controller_nodes),
|
'controller_nodes': str(controller_nodes),
|
||||||
'compute_nodes': str(compute_nodes),
|
'compute_nodes': str(compute_nodes),
|
||||||
'storage_nodes': str(storage_nodes),
|
'storage_nodes': str(storage_nodes),
|
||||||
'all_nodes': str(all_nodes)
|
'all_nodes': str(all_nodes)
|
||||||
})
|
}, force=True)
|
||||||
|
|
||||||
|
|
||||||
def get_marathon_framework(jvars):
|
def get_marathon_framework(jvars):
|
||||||
|
|
|
@ -465,37 +465,64 @@ def _load_variables_from_zk(zk):
|
||||||
return variables
|
return variables
|
||||||
|
|
||||||
|
|
||||||
|
class JvarsDict(dict):
|
||||||
|
"""Dict which can contain the 'global_vars' which are always preserved.
|
||||||
|
|
||||||
|
They cannot be be overriden by any update nor single item setting.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(JvarsDict, self).__init__(*args, **kwargs)
|
||||||
|
self.global_vars = {}
|
||||||
|
|
||||||
|
def __setitem__(self, key, value, force=False):
|
||||||
|
if not force and key in self.global_vars:
|
||||||
|
return
|
||||||
|
return super(JvarsDict, self).__setitem__(key, value)
|
||||||
|
|
||||||
|
def set_force(self, key, value):
|
||||||
|
"""Sets the variable even if it will override a global variable."""
|
||||||
|
return self.__setitem__(key, value, force=True)
|
||||||
|
|
||||||
|
def update(self, other_dict, force=False):
|
||||||
|
if not force:
|
||||||
|
other_dict = {key: value for key, value in other_dict.items()
|
||||||
|
if key not in self.global_vars}
|
||||||
|
super(JvarsDict, self).update(other_dict)
|
||||||
|
|
||||||
|
def set_global_vars(self, global_vars):
|
||||||
|
self.update(global_vars)
|
||||||
|
self.global_vars = global_vars
|
||||||
|
|
||||||
|
|
||||||
def _load_variables_from_file(service_dir, project_name):
|
def _load_variables_from_file(service_dir, project_name):
|
||||||
config_dir = os.path.join(service_dir, '..', 'config')
|
config_dir = os.path.join(service_dir, '..', 'config')
|
||||||
with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
|
jvars = JvarsDict()
|
||||||
global_vars = yaml.load(gf)
|
|
||||||
with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
|
with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
|
||||||
global_vars.update(yaml.load(gf))
|
jvars.set_global_vars(yaml.load(gf))
|
||||||
|
with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
|
||||||
|
jvars.update(yaml.load(gf))
|
||||||
|
# Apply the basic variables that aren't defined in any config file.
|
||||||
|
jvars.update({
|
||||||
|
'deployment_id': CONF.kolla.deployment_id,
|
||||||
|
'node_config_directory': '',
|
||||||
|
'timestamp': str(time.time())
|
||||||
|
})
|
||||||
|
# Get the exact marathon framework name.
|
||||||
|
config.get_marathon_framework(jvars)
|
||||||
# all.yml file uses some its variables to template itself by jinja2,
|
# all.yml file uses some its variables to template itself by jinja2,
|
||||||
# so its raw content is used to template the file
|
# so its raw content is used to template the file
|
||||||
all_yml_name = os.path.join(config_dir, 'all.yml')
|
all_yml_name = os.path.join(config_dir, 'all.yml')
|
||||||
with open(all_yml_name) as af:
|
jinja_utils.yaml_jinja_render(all_yml_name, jvars)
|
||||||
raw_vars = yaml.load(af)
|
# Apply the dynamic deployment variables.
|
||||||
raw_vars.update(global_vars)
|
config.apply_deployment_vars(jvars)
|
||||||
jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars))
|
|
||||||
jvars.update(global_vars)
|
|
||||||
|
|
||||||
proj_yml_name = os.path.join(config_dir, project_name,
|
proj_yml_name = os.path.join(config_dir, project_name,
|
||||||
'defaults', 'main.yml')
|
'defaults', 'main.yml')
|
||||||
if os.path.exists(proj_yml_name):
|
if os.path.exists(proj_yml_name):
|
||||||
proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
|
jinja_utils.yaml_jinja_render(proj_yml_name, jvars)
|
||||||
jvars))
|
|
||||||
jvars.update(proj_vars)
|
|
||||||
else:
|
else:
|
||||||
LOG.warning('Path missing %s' % proj_yml_name)
|
LOG.warning('Path missing %s' % proj_yml_name)
|
||||||
# Add deployment_id
|
|
||||||
jvars.update({'deployment_id': CONF.kolla.deployment_id})
|
|
||||||
# override node_config_directory to empty
|
|
||||||
jvars.update({'node_config_directory': ''})
|
|
||||||
# Add timestamp
|
|
||||||
jvars.update({'timestamp': str(time.time())})
|
|
||||||
config.apply_deployment_vars(jvars)
|
|
||||||
config.get_marathon_framework(jvars)
|
|
||||||
return jvars
|
return jvars
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,29 @@
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import collections
|
||||||
|
|
||||||
|
from kolla_mesos.common import jinja_utils
|
||||||
|
from kolla_mesos.tests import base
|
||||||
|
|
||||||
|
|
||||||
|
class TestJinjaUtils(base.BaseTestCase):
|
||||||
|
|
||||||
|
def test_dict_jinja_render(self):
|
||||||
|
raw_dict = collections.OrderedDict([
|
||||||
|
('first_key', '{{ test_var }}_test',),
|
||||||
|
('second_key', '{{ first_key }}_test'),
|
||||||
|
])
|
||||||
|
jvars = {'test_var': 'test'}
|
||||||
|
jinja_utils.dict_jinja_render(raw_dict, jvars)
|
||||||
|
self.assertEqual(jvars['first_key'], 'test_test')
|
||||||
|
self.assertEqual(jvars['second_key'], 'test_test_test')
|
|
@ -32,6 +32,37 @@ class TestAPI(base.BaseTestCase):
|
||||||
self.addCleanup(self.client.close)
|
self.addCleanup(self.client.close)
|
||||||
cfg.CONF.set_override('deployment_id', 'did', group='kolla')
|
cfg.CONF.set_override('deployment_id', 'did', group='kolla')
|
||||||
|
|
||||||
|
def test_jvars_dict(self):
|
||||||
|
jvars = service.JvarsDict(non_global1='old_value',
|
||||||
|
non_global2='old_value')
|
||||||
|
jvars.set_global_vars({'global1': 'old_value',
|
||||||
|
'global2': 'old_value'})
|
||||||
|
|
||||||
|
jvars.update({'global1': 'new_value',
|
||||||
|
'global2': 'new_value',
|
||||||
|
'non_global1': 'new_value',
|
||||||
|
'non_global2': 'new_value'})
|
||||||
|
self.assertDictEqual({'global1': 'old_value',
|
||||||
|
'global2': 'old_value',
|
||||||
|
'non_global1': 'new_value',
|
||||||
|
'non_global2': 'new_value'}, jvars)
|
||||||
|
|
||||||
|
jvars['global1'] = 'newer_value'
|
||||||
|
jvars['global2'] = 'newer_value'
|
||||||
|
jvars['non_global1'] = 'newer_value'
|
||||||
|
jvars['non_global2'] = 'newer_value'
|
||||||
|
self.assertDictEqual({'global1': 'old_value',
|
||||||
|
'global2': 'old_value',
|
||||||
|
'non_global1': 'newer_value',
|
||||||
|
'non_global2': 'newer_value'}, jvars)
|
||||||
|
|
||||||
|
jvars.set_force('global1', 'force_override')
|
||||||
|
jvars.update({'global2': 'force_override'}, force=True)
|
||||||
|
self.assertDictEqual({'global1': 'force_override',
|
||||||
|
'global2': 'force_override',
|
||||||
|
'non_global1': 'newer_value',
|
||||||
|
'non_global2': 'newer_value'}, jvars)
|
||||||
|
|
||||||
@mock.patch.object(service.config, 'get_marathon_framework')
|
@mock.patch.object(service.config, 'get_marathon_framework')
|
||||||
@mock.patch.object(service.config, 'apply_deployment_vars')
|
@mock.patch.object(service.config, 'apply_deployment_vars')
|
||||||
@mock.patch.object(service.MarathonApp, 'run')
|
@mock.patch.object(service.MarathonApp, 'run')
|
||||||
|
|
Loading…
Reference in New Issue