x
/
kolla-mesos
Code Issues Proposed changes

Parse and render yaml files in the proper order 

This commit introduces a clear order of loading jvars:
- passwords.yml
- globals.yml
- custom variables based on globals.yml
- config/<service>/all.yml
- config/<service/[...]

It also introduces a mechanism of jinja2 templating yaml
files by themselves, which means that one variable in yaml
file can be reused in the next variables inside the same file.

TrivialFix

Change-Id: I1a7a61e90963d396702f5e0c14eb476d3ccd21bf
This commit is contained in:
Michal Rostecki 2016-03-22 14:43:38 +01:00
parent 5666ef4ff5
commit ea39e9e5ed
7 changed files with 167 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
config/all.yml
View File

@ -126,6 +126,16 @@ openstack_region_name: "RegionOne"
# Valid options are [ novnc, spice ]
nova_console: "novnc"
####################
# Constraints
####################
controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]'
compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]'
controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]'
storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]'
####################
# Mesos-dns hosts
####################

8
etc/globals.yml
View File

@ -42,14 +42,6 @@ controller_nodes: "1"
compute_nodes: "1"
storage_nodes: "1"
####################
# Constraints
####################
controller_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "controller"]]'
compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "compute"]]'
controller_compute_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "LIKE", "(controller|compute)"]]'
storage_constraints: '[["hostname", "UNIQUE"], ["openstack_role", "CLUSTER", "storage"]]'
####################
# OpenStack options
####################

49
kolla_mesos/common/jinja_utils.py
View File

@ -10,17 +10,39 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import logging
import os
import jinja2
from jinja2 import meta
import six
import yaml
from kolla_mesos.common import type_utils
LOG = logging.getLogger(__name__)
# Customize PyYAML library to return the OrderedDict. That is needed, because
# when iterating on dict, we reuse its previous values when processing the
# next values and the order has to be preserved.
def ordered_dict_constructor(loader, node):
"""OrderedDict constructor for PyYAML."""
return collections.OrderedDict(loader.construct_pairs(node))
def ordered_dict_representer(dumper, data):
"""Representer for PyYAML which is able to work with OrderedDict."""
return dumper.represent_dict(data.items())
yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
ordered_dict_constructor)
yaml.add_representer(collections.OrderedDict, ordered_dict_representer)
def jinja_render(fullpath, global_config, extra=None):
variables = global_config
if extra:
@ -51,3 +73,30 @@ def jinja_find_required_variables(fullpath):
os.path.basename(fullpath))[0]
parsed_content = myenv.parse(template_source)
return meta.find_undeclared_variables(parsed_content)
def dict_jinja_render(raw_dict, jvars):
"""Renders dict with jinja2 using provided variables and itself.
By using itself, we mean reusing the previous values from dict for the
potential render of the next value in dict.
"""
for key, value in raw_dict.items():
if isinstance(value, six.string_types):
value = jinja_render_str(value, jvars)
elif isinstance(value, dict):
value = dict_jinja_render(value, jvars)
jvars[key] = value
def yaml_jinja_render(filename, jvars):
"""Parses YAML file and templates it with jinja2.
1. YAML file is rendered by jinja2 based on the provided variables.
2. Rendered file is parsed.
3. The every element dictionary being a result of parsing is rendered again
with itself.
"""
with open(filename, 'r') as yaml_file:
raw_dict = yaml.load(yaml_file)
dict_jinja_render(raw_dict, jvars)

4
kolla_mesos/configuration.py
View File

@ -155,13 +155,13 @@ def apply_deployment_vars(jvars):
'controller_compute_constraints':
controller_compute_constraints,
'storage_constraints': storage_constraints
})
}, force=True)
jvars.update({
'controller_nodes': str(controller_nodes),
'compute_nodes': str(compute_nodes),
'storage_nodes': str(storage_nodes),
'all_nodes': str(all_nodes)
})
}, force=True)
def get_marathon_framework(jvars):

65
kolla_mesos/service.py
View File

@ -465,37 +465,64 @@ def _load_variables_from_zk(zk):
return variables
class JvarsDict(dict):
"""Dict which can contain the 'global_vars' which are always preserved.
They cannot be be overriden by any update nor single item setting.
"""
def __init__(self, *args, **kwargs):
super(JvarsDict, self).__init__(*args, **kwargs)
self.global_vars = {}
def __setitem__(self, key, value, force=False):
if not force and key in self.global_vars:
return
return super(JvarsDict, self).__setitem__(key, value)
def set_force(self, key, value):
"""Sets the variable even if it will override a global variable."""
return self.__setitem__(key, value, force=True)
def update(self, other_dict, force=False):
if not force:
other_dict = {key: value for key, value in other_dict.items()
if key not in self.global_vars}
super(JvarsDict, self).update(other_dict)
def set_global_vars(self, global_vars):
self.update(global_vars)
self.global_vars = global_vars
def _load_variables_from_file(service_dir, project_name):
config_dir = os.path.join(service_dir, '..', 'config')
with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
global_vars = yaml.load(gf)
jvars = JvarsDict()
with open(file_utils.find_config_file('globals.yml'), 'r') as gf:
global_vars.update(yaml.load(gf))
jvars.set_global_vars(yaml.load(gf))
with open(file_utils.find_config_file('passwords.yml'), 'r') as gf:
jvars.update(yaml.load(gf))
# Apply the basic variables that aren't defined in any config file.
jvars.update({
'deployment_id': CONF.kolla.deployment_id,
'node_config_directory': '',
'timestamp': str(time.time())
})
# Get the exact marathon framework name.
config.get_marathon_framework(jvars)
# all.yml file uses some its variables to template itself by jinja2,
# so its raw content is used to template the file
all_yml_name = os.path.join(config_dir, 'all.yml')
with open(all_yml_name) as af:
raw_vars = yaml.load(af)
raw_vars.update(global_vars)
jvars = yaml.load(jinja_utils.jinja_render(all_yml_name, raw_vars))
jvars.update(global_vars)
jinja_utils.yaml_jinja_render(all_yml_name, jvars)
# Apply the dynamic deployment variables.
config.apply_deployment_vars(jvars)
proj_yml_name = os.path.join(config_dir, project_name,
'defaults', 'main.yml')
if os.path.exists(proj_yml_name):
proj_vars = yaml.load(jinja_utils.jinja_render(proj_yml_name,
jvars))
jvars.update(proj_vars)
jinja_utils.yaml_jinja_render(proj_yml_name, jvars)
else:
LOG.warning('Path missing %s' % proj_yml_name)
# Add deployment_id
jvars.update({'deployment_id': CONF.kolla.deployment_id})
# override node_config_directory to empty
jvars.update({'node_config_directory': ''})
# Add timestamp
jvars.update({'timestamp': str(time.time())})
config.apply_deployment_vars(jvars)
config.get_marathon_framework(jvars)
return jvars

29
kolla_mesos/tests/common/test_jinja_utils.py Normal file
View File

@ -0,0 +1,29 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
from kolla_mesos.common import jinja_utils
from kolla_mesos.tests import base
class TestJinjaUtils(base.BaseTestCase):
def test_dict_jinja_render(self):
raw_dict = collections.OrderedDict([
('first_key', '{{ test_var }}_test',),
('second_key', '{{ first_key }}_test'),
])
jvars = {'test_var': 'test'}
jinja_utils.dict_jinja_render(raw_dict, jvars)
self.assertEqual(jvars['first_key'], 'test_test')
self.assertEqual(jvars['second_key'], 'test_test_test')

31
kolla_mesos/tests/test_service.py
View File

@ -32,6 +32,37 @@ class TestAPI(base.BaseTestCase):
self.addCleanup(self.client.close)
cfg.CONF.set_override('deployment_id', 'did', group='kolla')
def test_jvars_dict(self):
jvars = service.JvarsDict(non_global1='old_value',
non_global2='old_value')
jvars.set_global_vars({'global1': 'old_value',
'global2': 'old_value'})
jvars.update({'global1': 'new_value',
'global2': 'new_value',
'non_global1': 'new_value',
'non_global2': 'new_value'})
self.assertDictEqual({'global1': 'old_value',
'global2': 'old_value',
'non_global1': 'new_value',
'non_global2': 'new_value'}, jvars)
jvars['global1'] = 'newer_value'
jvars['global2'] = 'newer_value'
jvars['non_global1'] = 'newer_value'
jvars['non_global2'] = 'newer_value'
self.assertDictEqual({'global1': 'old_value',
'global2': 'old_value',
'non_global1': 'newer_value',
'non_global2': 'newer_value'}, jvars)
jvars.set_force('global1', 'force_override')
jvars.update({'global2': 'force_override'}, force=True)
self.assertDictEqual({'global1': 'force_override',
'global2': 'force_override',
'non_global1': 'newer_value',
'non_global2': 'newer_value'}, jvars)
@mock.patch.object(service.config, 'get_marathon_framework')
@mock.patch.object(service.config, 'apply_deployment_vars')
@mock.patch.object(service.MarathonApp, 'run')