x/fuel-ccp
Code Issues Proposed changes

Improvement of k8s Deployments update

Browse Source 
* Deployment will be updated only if service-related configuration
  was changed
* SilentUndefined handler to ignore undefined variables of jinja
  templates was added

Change-Id: Iad384ee447268cba44cbfacba81118ec44ca31c3
This commit is contained in:
Andrey Pavlov 2016-09-16 13:08:55 +03:00
parent 6a864bb4ba
commit ae770e795f
5 changed files with 105 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
fuel_ccp/common/jinja_utils.py
View File

@ -3,13 +3,31 @@ import os
import jinja2
class SilentUndefined(jinja2.Undefined):
def _fail_with_undefined_error(self, *args, **kwargs):
return ''
__add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \
__truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \
__mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \
__getitem__ = __lt__ = __le__ = __gt__ = __ge__ = __int__ = \
__float__ = __complex__ = __pow__ = __rpow__ = \
_fail_with_undefined_error
def str_to_bool(text):
return text is not None and text.lower() in ['true', 'yes']
def jinja_render(path, context, functions=()):
def jinja_render(path, context, functions=(), ignore_undefined=False):
kwargs = {}
if ignore_undefined:
kwargs['undefined'] = SilentUndefined
else:
kwargs['undefined'] = jinja2.StrictUndefined
env = jinja2.Environment(loader=jinja2.FileSystemLoader(
os.path.dirname(path)), undefined=jinja2.StrictUndefined)
os.path.dirname(path)), **kwargs)
env.filters['bool'] = str_to_bool
for func in functions:
env.globals[func.__name__] = func

40
fuel_ccp/deploy.py
View File

@ -1,8 +1,10 @@
import hashlib
import json
import logging
import os
import re
from fuel_ccp.common import jinja_utils
from fuel_ccp.common import utils
from fuel_ccp import config
from fuel_ccp import kubernetes
@ -31,14 +33,29 @@ def _expand_files(service, files):
_expand(cmd)
def _get_configmaps_version(configmaps):
"""Get concatenation of ConfigMaps versions
def _get_configmaps_version(configmaps, service_dir, files, configs):
"""Get overall ConfigMaps version
If version of any of the ConfigMaps changed, the overall version will be
If any of the ConfigMaps changed, the overall version will be
changed and deployment will be updated no matter if the deployment spec
was updated or not.
"""
return ''.join(cm.obj['metadata']['resourceVersion'] for cm in configmaps)
versions = ''.join(cm.obj['metadata']['resourceVersion']
for cm in configmaps)
files_hash = _get_service_files_hash(service_dir, files, configs)
return versions + files_hash
def _get_service_files_hash(service_dir, files, configs):
data = {}
if files:
for filename, f in files.items():
path = os.path.join(service_dir, "files", f["content"])
data[filename] = jinja_utils.jinja_render(
path, configs, ignore_undefined=True)
dump = json.dumps(data, sort_keys=True).encode("utf-8")
return hashlib.sha1(dump).hexdigest()
def parse_role(service_dir, role, config):
@ -58,13 +75,16 @@ def parse_role(service_dir, role, config):
workflow_cm = _create_workflow(workflows, service["name"])
configmaps = config['configmaps'] + (files_cm, meta_cm, workflow_cm)
cm_version = _get_configmaps_version(
configmaps, service_dir, role.get("files"), config['configs'])
for cont in service["containers"]:
daemon_cmd = cont["daemon"]
daemon_cmd["name"] = cont["name"]
_create_pre_jobs(service, cont)
_create_post_jobs(service, cont)
cont['cm_version'] = _get_configmaps_version(configmaps)
cont['cm_version'] = cm_version
cont_spec = templates.serialize_daemon_pod_spec(service)
affinity = templates.serialize_affinity(service, config["topology"])
@ -245,11 +265,11 @@ def _create_start_script_configmap():
return kubernetes.process_object(cm)
def _create_files_configmap(service_dir, service_name, configs):
def _create_files_configmap(service_dir, service_name, files):
configmap_name = "%s-%s" % (service_name, templates.FILES_CONFIG)
data = {}
if configs:
for filename, f in configs.items():
if files:
for filename, f in files.items():
with open(os.path.join(
service_dir, "files", f["content"]), "r") as f:
data[filename] = f.read()
@ -351,9 +371,9 @@ def deploy_components(components=None):
namespace = CONF.kubernetes.namespace
_create_namespace(namespace)
globals_cm = _create_globals_configmap(config["configs"])
_create_globals_configmap(config["configs"])
start_script_cm = _create_start_script_configmap()
config['configmaps'] = (globals_cm, start_script_cm)
config['configmaps'] = (start_script_cm,)
for component in components:
parse_role(components_map[component]['service_dir'],

2
fuel_ccp/tests/common/example.j2
View File

@ -1,3 +1,5 @@
{{ base_distro }}
{{ base_tag }}
{{ maintainer }}
{{ duck["egg"] }}
{{ duck.egg }}

44
fuel_ccp/tests/common/test_jinja_utils.py
View File

@ -1,11 +1,12 @@
import os
import sys
from jinja2 import exceptions
from fuel_ccp.common import jinja_utils
from fuel_ccp.common import utils
from fuel_ccp.tests import base
class TestJinjaUtils(base.TestCase):
filename = utils.get_resource_path('tests/common/example.j2')
def test_str_to_bool(self):
self.assertTrue(jinja_utils.str_to_bool('true'))
@ -14,14 +15,39 @@ class TestJinjaUtils(base.TestCase):
self.assertFalse(jinja_utils.str_to_bool('no'))
self.assertFalse(jinja_utils.str_to_bool('some_random_string'))
def test_jinja_render(self):
filename = os.path.join(
os.path.dirname(sys.modules[__name__].__file__),
'example.j2')
def test_jinja_render_strict(self):
context = {
"base_distro": "debian",
"base_tag": "jessie",
"maintainer": "some maintainer"
"maintainer": "some maintainer",
"duck": {"egg": "needle"}
}
content = jinja_utils.jinja_render(filename, context)
self.assertEqual("debian\njessie\nsome maintainer", content)
content = jinja_utils.jinja_render(self.filename, context)
self.assertEqual(
"debian\njessie\nsome maintainer\nneedle\nneedle", content)
context = {
"base_distro": "debian"
}
self.assertRaises(exceptions.UndefinedError, jinja_utils.jinja_render,
self.filename, context)
def test_jinja_render_silent(self):
context = {
"base_distro": "debian",
"base_tag": "jessie",
"maintainer": "some maintainer",
"duck": {"egg": "needle"}
}
content = jinja_utils.jinja_render(
self.filename, context, ignore_undefined=True)
self.assertEqual(
"debian\njessie\nsome maintainer\nneedle\nneedle", content)
context = {
"base_distro": "debian"
}
content = jinja_utils.jinja_render(
self.filename, context, ignore_undefined=True)
self.assertEqual(
"debian\n\n\n\n", content)

20
fuel_ccp/tests/test_deploy.py
View File

@ -124,12 +124,28 @@ class TestDeploy(base.TestCase):
self.assertTrue(result)
def test_get_configmaps_version(self):
self.useFixture(fixtures.MockPatch(
"fuel_ccp.deploy._get_service_files_hash", return_value='222'))
cm_list = [mock.Mock(obj={'metadata': {'resourceVersion': '1'}})
for _ in range(3)]
self.assertEqual('111', deploy._get_configmaps_version(cm_list))
self.assertEqual('111222', deploy._get_configmaps_version(
cm_list, mock.ANY, mock.ANY, mock.ANY))
cm_list = []
self.assertEqual('', deploy._get_configmaps_version(cm_list))
self.assertEqual('222', deploy._get_configmaps_version(
cm_list, mock.ANY, mock.ANY, mock.ANY))
def test_get_service_files_hash(self):
files = {
'file': {'content': '/tmp/file'}
}
self.useFixture(fixtures.MockPatch(
"fuel_ccp.common.jinja_utils.jinja_render",
return_value='rendered'))
expected_hash = '86e85bd63aef5a740d4b7b887ade37ec9017c961'
self.assertEqual(
expected_hash, deploy._get_service_files_hash('/tmp', files, {}))
class TestDeployCreateService(base.TestCase):