Merge "Introduce cross-repository config templating"
This commit is contained in:
commit
074fbf089e
@ -1,4 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import socket
|
import socket
|
||||||
|
|
||||||
import jinja2
|
import jinja2
|
||||||
@ -41,3 +42,17 @@ def jinja_render(path, context, functions=(), ignore_undefined=False):
|
|||||||
env.globals[func.__name__] = func
|
env.globals[func.__name__] = func
|
||||||
content = env.get_template(os.path.basename(path)).render(context)
|
content = env.get_template(os.path.basename(path)).render(context)
|
||||||
return content
|
return content
|
||||||
|
|
||||||
|
|
||||||
|
def generate_jinja_imports(filenames):
|
||||||
|
"""Generate str of jinja imports from list of filenames."""
|
||||||
|
imports = [] # list of j2 imports: "{% import 'msg.j2' as msg %}"
|
||||||
|
for name in filenames:
|
||||||
|
import_as, extension = os.path.splitext(name) # remove file extension
|
||||||
|
if not re.match('[a-zA-Z_][a-zA-Z0-9_]*', import_as):
|
||||||
|
raise RuntimeError('Wrong templates file naming: the %s cannot be '
|
||||||
|
'imported by jinja with %s name. Please use '
|
||||||
|
'python compatible naming' % (name, import_as))
|
||||||
|
imports.append(
|
||||||
|
"{% import '" + name + "' as " + import_as + " with context %}")
|
||||||
|
return ''.join(imports)
|
||||||
|
@ -76,6 +76,27 @@ def address(service, port=None, external=False, with_scheme=False):
|
|||||||
return addr
|
return addr
|
||||||
|
|
||||||
|
|
||||||
|
def get_repositories_exports(repos_names=None):
|
||||||
|
"""Load shared templates from ./export dirs of the repositories. """
|
||||||
|
exports = dict()
|
||||||
|
repos_names = repos_names or [d['name'] for d in CONF.repositories.repos]
|
||||||
|
for repo in repos_names:
|
||||||
|
exports_dir = os.path.join(CONF.repositories.path, repo, 'exports')
|
||||||
|
if os.path.exists(exports_dir) and os.path.isdir(exports_dir):
|
||||||
|
for export in os.listdir(exports_dir):
|
||||||
|
path = os.path.join(exports_dir, export)
|
||||||
|
LOG.debug('Found shared jinja template file %s', path)
|
||||||
|
if export not in exports:
|
||||||
|
exports[export] = list()
|
||||||
|
with open(path) as f:
|
||||||
|
exports[export].append(f.read())
|
||||||
|
|
||||||
|
for export in exports:
|
||||||
|
exports[export] = '\n'.join(exports[export])
|
||||||
|
|
||||||
|
return exports
|
||||||
|
|
||||||
|
|
||||||
def get_deploy_components_info(rendering_context=None):
|
def get_deploy_components_info(rendering_context=None):
|
||||||
if rendering_context is None:
|
if rendering_context is None:
|
||||||
rendering_context = CONF.configs._dict
|
rendering_context = CONF.configs._dict
|
||||||
|
@ -71,7 +71,7 @@ def process_files(files, service_dir):
|
|||||||
f["content"] = content
|
f["content"] = content
|
||||||
|
|
||||||
|
|
||||||
def parse_role(component, topology, configmaps):
|
def parse_role(component, topology, configmaps, jinja_imports):
|
||||||
service_dir = component["service_dir"]
|
service_dir = component["service_dir"]
|
||||||
role = component["service_content"]
|
role = component["service_content"]
|
||||||
component_name = component["component_name"]
|
component_name = component["component_name"]
|
||||||
@ -82,7 +82,8 @@ def parse_role(component, topology, configmaps):
|
|||||||
_expand_files(service, role.get("files"))
|
_expand_files(service, role.get("files"))
|
||||||
|
|
||||||
process_files(role.get("files"), service_dir)
|
process_files(role.get("files"), service_dir)
|
||||||
files_cm = _create_files_configmap(service_name, role.get("files"))
|
files_cm = _create_files_configmap(service_name, role.get("files"),
|
||||||
|
jinja_imports)
|
||||||
meta_cm = _create_meta_configmap(service)
|
meta_cm = _create_meta_configmap(service)
|
||||||
|
|
||||||
workflows = _parse_workflows(service)
|
workflows = _parse_workflows(service)
|
||||||
@ -311,13 +312,13 @@ def _create_start_script_configmap():
|
|||||||
return kubernetes.process_object(cm)
|
return kubernetes.process_object(cm)
|
||||||
|
|
||||||
|
|
||||||
def _create_files_configmap(service_name, files):
|
def _create_files_configmap(service_name, files, macros_imports):
|
||||||
configmap_name = "%s-%s" % (service_name, templates.FILES_CONFIG)
|
configmap_name = "%s-%s" % (service_name, templates.FILES_CONFIG)
|
||||||
data = {}
|
data = {}
|
||||||
if files:
|
if files:
|
||||||
for filename, f in files.items():
|
for filename, f in files.items():
|
||||||
with open(f["content"], "r") as f:
|
with open(f["content"], "r") as f:
|
||||||
data[filename] = f.read()
|
data[filename] = macros_imports + f.read()
|
||||||
data["placeholder"] = ""
|
data["placeholder"] = ""
|
||||||
template = templates.serialize_configmap(configmap_name, data)
|
template = templates.serialize_configmap(configmap_name, data)
|
||||||
return kubernetes.process_object(template)
|
return kubernetes.process_object(template)
|
||||||
@ -334,6 +335,13 @@ def _create_meta_configmap(service):
|
|||||||
return kubernetes.process_object(template)
|
return kubernetes.process_object(template)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_jinja_templates_configmap(templates_files):
|
||||||
|
"""Create config map of files from fuel-ccp-repo/exports dirs."""
|
||||||
|
serialized = templates.serialize_configmap(templates.EXPORTS_CONFIG,
|
||||||
|
templates_files)
|
||||||
|
return kubernetes.process_object(serialized)
|
||||||
|
|
||||||
|
|
||||||
def _make_topology(nodes, roles, replicas):
|
def _make_topology(nodes, roles, replicas):
|
||||||
failed = False
|
failed = False
|
||||||
# TODO(sreshetniak): move it to validation
|
# TODO(sreshetniak): move it to validation
|
||||||
@ -443,7 +451,8 @@ def check_images_change(objects):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology):
|
def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology,
|
||||||
|
jinja_imports):
|
||||||
from_version = upgrade_data['_meta']['from']
|
from_version = upgrade_data['_meta']['from']
|
||||||
to_version = upgrade_data['_meta']['to']
|
to_version = upgrade_data['_meta']['to']
|
||||||
component = upgrade_data['_meta']['component']
|
component = upgrade_data['_meta']['component']
|
||||||
@ -457,7 +466,7 @@ def create_upgrade_jobs(component_name, upgrade_data, configmaps, topology):
|
|||||||
step['files'] = {f: files[f] for f in step['files']}
|
step['files'] = {f: files[f] for f in step['files']}
|
||||||
|
|
||||||
process_files(files, component['service_dir'])
|
process_files(files, component['service_dir'])
|
||||||
_create_files_configmap(prefix, files)
|
_create_files_configmap(prefix, files, jinja_imports)
|
||||||
container = {
|
container = {
|
||||||
"name": prefix,
|
"name": prefix,
|
||||||
"pre": [],
|
"pre": [],
|
||||||
@ -541,14 +550,17 @@ def deploy_components(components_map, components):
|
|||||||
_create_namespace(CONF.configs)
|
_create_namespace(CONF.configs)
|
||||||
_create_globals_configmap(CONF.configs)
|
_create_globals_configmap(CONF.configs)
|
||||||
start_script_cm = _create_start_script_configmap()
|
start_script_cm = _create_start_script_configmap()
|
||||||
configmaps = (start_script_cm,)
|
|
||||||
|
# Create cm with jinja config templates shared across all repositories.
|
||||||
|
templates_files = utils.get_repositories_exports()
|
||||||
|
jinja_imports = jinja_utils.generate_jinja_imports(templates_files.keys())
|
||||||
|
templates_cm = _create_jinja_templates_configmap(templates_files)
|
||||||
|
configmaps = (start_script_cm, templates_cm)
|
||||||
|
|
||||||
upgrading_components = {}
|
upgrading_components = {}
|
||||||
for service_name in components:
|
for service_name in components:
|
||||||
service = components_map[service_name]
|
service = components_map[service_name]
|
||||||
objects_gen = parse_role(service,
|
objects_gen = parse_role(service, topology, configmaps, jinja_imports)
|
||||||
topology=topology,
|
|
||||||
configmaps=configmaps)
|
|
||||||
objects = list(itertools.chain.from_iterable(objects_gen))
|
objects = list(itertools.chain.from_iterable(objects_gen))
|
||||||
component_name = service['component_name']
|
component_name = service['component_name']
|
||||||
do_upgrade = component_name in upgrading_components
|
do_upgrade = component_name in upgrading_components
|
||||||
@ -578,8 +590,9 @@ def deploy_components(components_map, components):
|
|||||||
upgrading_components[component_name][service_name] = objects
|
upgrading_components[component_name][service_name] = objects
|
||||||
|
|
||||||
for component_name, component_upg in upgrading_components.items():
|
for component_name, component_upg in upgrading_components.items():
|
||||||
create_upgrade_jobs(component_name, component_upg, configmaps,
|
create_upgrade_jobs(component_name, component_upg,
|
||||||
topology)
|
configmaps, topology,
|
||||||
|
jinja_imports)
|
||||||
|
|
||||||
if 'keystone' in components:
|
if 'keystone' in components:
|
||||||
_create_openrc(CONF.configs)
|
_create_openrc(CONF.configs)
|
||||||
|
@ -11,6 +11,7 @@ SCRIPT_CONFIG = "start-script"
|
|||||||
FILES_CONFIG = "files"
|
FILES_CONFIG = "files"
|
||||||
META_CONFIG = "meta"
|
META_CONFIG = "meta"
|
||||||
ROLE_CONFIG = "role"
|
ROLE_CONFIG = "role"
|
||||||
|
EXPORTS_CONFIG = "exports"
|
||||||
|
|
||||||
ENTRYPOINT_PATH = "/opt/ccp_start_script/bin/start_script.py"
|
ENTRYPOINT_PATH = "/opt/ccp_start_script/bin/start_script.py"
|
||||||
PYTHON_PATH = "/usr/bin/python"
|
PYTHON_PATH = "/usr/bin/python"
|
||||||
@ -68,6 +69,10 @@ def serialize_volume_mounts(container, for_job=None):
|
|||||||
"name": SCRIPT_CONFIG,
|
"name": SCRIPT_CONFIG,
|
||||||
"mountPath": "/opt/ccp_start_script/bin"
|
"mountPath": "/opt/ccp_start_script/bin"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": EXPORTS_CONFIG,
|
||||||
|
"mountPath": "/etc/ccp/%s" % EXPORTS_CONFIG
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": FILES_CONFIG,
|
"name": FILES_CONFIG,
|
||||||
"mountPath": "/etc/ccp/%s" % FILES_CONFIG
|
"mountPath": "/etc/ccp/%s" % FILES_CONFIG
|
||||||
@ -250,10 +255,16 @@ def serialize_volumes(service, for_job=None):
|
|||||||
"name": "%s-%s" % (service["name"], FILES_CONFIG),
|
"name": "%s-%s" % (service["name"], FILES_CONFIG),
|
||||||
"items": file_items
|
"items": file_items
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": EXPORTS_CONFIG,
|
||||||
|
"configMap": {
|
||||||
|
"name": EXPORTS_CONFIG,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
volume_names = [GLOBAL_CONFIG, META_CONFIG, ROLE_CONFIG, SCRIPT_CONFIG,
|
volume_names = [GLOBAL_CONFIG, META_CONFIG, ROLE_CONFIG, SCRIPT_CONFIG,
|
||||||
FILES_CONFIG]
|
FILES_CONFIG, EXPORTS_CONFIG]
|
||||||
for cont in itertools.chain(service["containers"], [for_job]):
|
for cont in itertools.chain(service["containers"], [for_job]):
|
||||||
for v in cont.get("volumes", ()):
|
for v in cont.get("volumes", ()):
|
||||||
if v["name"] in volume_names:
|
if v["name"] in volume_names:
|
||||||
|
@ -44,6 +44,7 @@ class TestDeploy(base.TestCase):
|
|||||||
{'mountPath': '/etc/ccp/meta', 'name': 'meta'},
|
{'mountPath': '/etc/ccp/meta', 'name': 'meta'},
|
||||||
{'mountPath': '/opt/ccp_start_script/bin',
|
{'mountPath': '/opt/ccp_start_script/bin',
|
||||||
'name': 'start-script'},
|
'name': 'start-script'},
|
||||||
|
{'mountPath': '/etc/ccp/exports', 'name': 'exports'},
|
||||||
{'mountPath': '/etc/ccp/files', 'name': 'files'}
|
{'mountPath': '/etc/ccp/files', 'name': 'files'}
|
||||||
],
|
],
|
||||||
"readinessProbe": {
|
"readinessProbe": {
|
||||||
|
Loading…
Reference in New Issue
Block a user