diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index fc8b41e..71b066f 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -20,3 +20,4 @@ include: - contrib.python - contrib.charmsupport - contrib.openstack.policyd + - contrib.templating diff --git a/charmhelpers/contrib/templating/__init__.py b/charmhelpers/contrib/templating/__init__.py new file mode 100644 index 0000000..d7567b8 --- /dev/null +++ b/charmhelpers/contrib/templating/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/charmhelpers/contrib/templating/contexts.py b/charmhelpers/contrib/templating/contexts.py new file mode 100644 index 0000000..c1adf94 --- /dev/null +++ b/charmhelpers/contrib/templating/contexts.py @@ -0,0 +1,137 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Copyright 2013 Canonical Ltd. +# +# Authors: +# Charm Helpers Developers +"""A helper to create a yaml cache of config with namespaced relation data.""" +import os +import yaml + +import six + +import charmhelpers.core.hookenv + + +charm_dir = os.environ.get('CHARM_DIR', '') + + +def dict_keys_without_hyphens(a_dict): + """Return the a new dict with underscores instead of hyphens in keys.""" + return dict( + (key.replace('-', '_'), val) for key, val in a_dict.items()) + + +def update_relations(context, namespace_separator=':'): + """Update the context with the relation data.""" + # Add any relation data prefixed with the relation type. + relation_type = charmhelpers.core.hookenv.relation_type() + relations = [] + context['current_relation'] = {} + if relation_type is not None: + relation_data = charmhelpers.core.hookenv.relation_get() + context['current_relation'] = relation_data + # Deprecated: the following use of relation data as keys + # directly in the context will be removed. + relation_data = dict( + ("{relation_type}{namespace_separator}{key}".format( + relation_type=relation_type, + key=key, + namespace_separator=namespace_separator), val) + for key, val in relation_data.items()) + relation_data = dict_keys_without_hyphens(relation_data) + context.update(relation_data) + relations = charmhelpers.core.hookenv.relations_of_type(relation_type) + relations = [dict_keys_without_hyphens(rel) for rel in relations] + + context['relations_full'] = charmhelpers.core.hookenv.relations() + + # the hookenv.relations() data structure is effectively unusable in + # templates and other contexts when trying to access relation data other + # than the current relation. So provide a more useful structure that works + # with any hook. + local_unit = charmhelpers.core.hookenv.local_unit() + relations = {} + for rname, rids in context['relations_full'].items(): + relations[rname] = [] + for rid, rdata in rids.items(): + data = rdata.copy() + if local_unit in rdata: + data.pop(local_unit) + for unit_name, rel_data in data.items(): + new_data = {'__relid__': rid, '__unit__': unit_name} + new_data.update(rel_data) + relations[rname].append(new_data) + context['relations'] = relations + + +def juju_state_to_yaml(yaml_path, namespace_separator=':', + allow_hyphens_in_keys=True, mode=None): + """Update the juju config and state in a yaml file. + + This includes any current relation-get data, and the charm + directory. + + This function was created for the ansible and saltstack + support, as those libraries can use a yaml file to supply + context to templates, but it may be useful generally to + create and update an on-disk cache of all the config, including + previous relation data. + + By default, hyphens are allowed in keys as this is supported + by yaml, but for tools like ansible, hyphens are not valid [1]. + + [1] http://www.ansibleworks.com/docs/playbooks_variables.html#what-makes-a-valid-variable-name + """ + config = charmhelpers.core.hookenv.config() + + # Add the charm_dir which we will need to refer to charm + # file resources etc. + config['charm_dir'] = charm_dir + config['local_unit'] = charmhelpers.core.hookenv.local_unit() + config['unit_private_address'] = charmhelpers.core.hookenv.unit_private_ip() + config['unit_public_address'] = charmhelpers.core.hookenv.unit_get( + 'public-address' + ) + + # Don't use non-standard tags for unicode which will not + # work when salt uses yaml.load_safe. + yaml.add_representer(six.text_type, + lambda dumper, value: dumper.represent_scalar( + six.u('tag:yaml.org,2002:str'), value)) + + yaml_dir = os.path.dirname(yaml_path) + if not os.path.exists(yaml_dir): + os.makedirs(yaml_dir) + + if os.path.exists(yaml_path): + with open(yaml_path, "r") as existing_vars_file: + existing_vars = yaml.load(existing_vars_file.read()) + else: + with open(yaml_path, "w+"): + pass + existing_vars = {} + + if mode is not None: + os.chmod(yaml_path, mode) + + if not allow_hyphens_in_keys: + config = dict_keys_without_hyphens(config) + existing_vars.update(config) + + update_relations(existing_vars, namespace_separator) + + with open(yaml_path, "w+") as fp: + fp.write(yaml.dump(existing_vars, default_flow_style=False)) diff --git a/charmhelpers/contrib/templating/jinja.py b/charmhelpers/contrib/templating/jinja.py new file mode 100644 index 0000000..c6ad9d0 --- /dev/null +++ b/charmhelpers/contrib/templating/jinja.py @@ -0,0 +1,51 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Templating using the python-jinja2 package. +""" +import six +from charmhelpers.fetch import apt_install, apt_update +try: + import jinja2 +except ImportError: + apt_update(fatal=True) + if six.PY3: + apt_install(["python3-jinja2"], fatal=True) + else: + apt_install(["python-jinja2"], fatal=True) + import jinja2 + + +DEFAULT_TEMPLATES_DIR = 'templates' + + +def render(template_name, context, template_dir=DEFAULT_TEMPLATES_DIR, + jinja_env_args=None): + """ + Render jinja2 template with provided context. + + :param template_name: name of the jinja template file + :param context: template context + :param template_dir: directory in which the template file is located + :param jinja_env_args: additional arguments passed to the + jinja2.Environment. Expected dict with format + {'arg_name': 'arg_value'} + :return: Rendered template as a string + """ + env_kwargs = jinja_env_args or {} + templates = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_dir), **env_kwargs) + template = templates.get_template(template_name) + return template.render(context) diff --git a/charmhelpers/contrib/templating/pyformat.py b/charmhelpers/contrib/templating/pyformat.py new file mode 100644 index 0000000..51a24dc --- /dev/null +++ b/charmhelpers/contrib/templating/pyformat.py @@ -0,0 +1,27 @@ +# Copyright 2014-2015 Canonical Limited. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +Templating using standard Python str.format() method. +''' + +from charmhelpers.core import hookenv + + +def render(template, extra={}, **kwargs): + """Return the template rendered using Python's str.format().""" + context = hookenv.execution_environment() + context.update(extra) + context.update(kwargs) + return template.format(**context) diff --git a/hooks/utils.py b/hooks/utils.py index 508815a..3acff3a 100644 --- a/hooks/utils.py +++ b/hooks/utils.py @@ -61,6 +61,7 @@ from charmhelpers.contrib.openstack.utils import ( from charmhelpers.contrib.openstack.ha.utils import ( assert_charm_supports_dns_ha ) +from charmhelpers.contrib.templating.jinja import render from charmhelpers.core.host import ( mkdir, rsync, @@ -85,10 +86,8 @@ from charmhelpers.contrib.network import ip as utils import netifaces from netaddr import IPNetwork -import jinja2 -TEMPLATES_DIR = 'templates' COROSYNC_CONF = '/etc/corosync/corosync.conf' COROSYNC_DEFAULT = '/etc/default/corosync' COROSYNC_AUTHKEY = '/etc/corosync/authkey' @@ -346,8 +345,8 @@ def emit_systemd_overrides_file(): os.mkdir(overrides_dir) write_file(path=overrides_file, - content=render_template('systemd-overrides.conf', - systemd_overrides_context)) + content=render('systemd-overrides.conf', + systemd_overrides_context)) # Update systemd with the new information subprocess.check_call(['systemctl', 'daemon-reload']) @@ -357,8 +356,7 @@ def emit_corosync_conf(): corosync_conf_context = get_corosync_conf() if corosync_conf_context: write_file(path=COROSYNC_CONF, - content=render_template('corosync.conf', - corosync_conf_context)) + content=render('corosync.conf', corosync_conf_context)) return True return False @@ -376,11 +374,10 @@ def emit_base_conf(): os.mkdir(PCMKR_CONFIG_DIR) corosync_default_context = {'corosync_enabled': 'yes'} write_file(path=COROSYNC_DEFAULT, - content=render_template('corosync', - corosync_default_context)) + content=render('corosync', corosync_default_context)) write_file(path=COROSYNC_HACLUSTER_ACL, - content=render_template('hacluster.acl', {})) + content=render('hacluster.acl', {})) corosync_key = config('corosync_key') if corosync_key: @@ -398,14 +395,6 @@ def emit_base_conf(): return False -def render_template(template_name, context, template_dir=TEMPLATES_DIR): - templates = jinja2.Environment( - loader=jinja2.FileSystemLoader(template_dir) - ) - template = templates.get_template(template_name) - return template.render(context) - - def assert_charm_supports_ipv6(): """Check whether we are able to support charms ipv6.""" _release = lsb_release()['DISTRIB_CODENAME'].lower() diff --git a/test-requirements.txt b/test-requirements.txt index 0aabe17..823463d 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -37,7 +37,8 @@ git+https://github.com/openstack-charmers/zaza.git#egg=zaza git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack # Needed for charm-glance: -git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6' +git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.8' +tempest<31.0.0;python_version<'3.8' tempest<24.0.0;python_version<'3.6' croniter # needed for charm-rabbitmq-server unit tests diff --git a/unit_tests/test_hacluster_utils.py b/unit_tests/test_hacluster_utils.py index 45a4bd6..d6f58fb 100644 --- a/unit_tests/test_hacluster_utils.py +++ b/unit_tests/test_hacluster_utils.py @@ -320,13 +320,13 @@ class UtilsTestCase(unittest.TestCase): @mock.patch.object(subprocess, 'check_call') @mock.patch.object(utils.os, 'mkdir') @mock.patch.object(utils.os.path, 'exists') - @mock.patch.object(utils, 'render_template') + @mock.patch.object(utils, 'render') @mock.patch.object(utils, 'write_file') @mock.patch.object(utils, 'is_unit_paused_set') @mock.patch.object(utils, 'config') def test_emit_systemd_overrides_file(self, mock_config, mock_is_unit_paused_set, - mock_write_file, mock_render_template, + mock_write_file, mock_render, mock_path_exists, mock_mkdir, mock_check_call): @@ -339,13 +339,13 @@ class UtilsTestCase(unittest.TestCase): mock_path_exists.return_value = True utils.emit_systemd_overrides_file() self.assertEqual(2, len(mock_write_file.mock_calls)) - mock_render_template.assert_has_calls( + mock_render.assert_has_calls( [mock.call('systemd-overrides.conf', cfg), mock.call('systemd-overrides.conf', cfg)]) mock_check_call.assert_has_calls([mock.call(['systemctl', 'daemon-reload'])]) mock_write_file.mock_calls = [] - mock_render_template.mock_calls = [] + mock_render.mock_calls = [] mock_check_call.mock_calls = [] # Disable timeout @@ -358,7 +358,7 @@ class UtilsTestCase(unittest.TestCase): mock_path_exists.return_value = True utils.emit_systemd_overrides_file() self.assertEqual(2, len(mock_write_file.mock_calls)) - mock_render_template.assert_has_calls( + mock_render.assert_has_calls( [mock.call('systemd-overrides.conf', expected_cfg), mock.call('systemd-overrides.conf', expected_cfg)]) mock_check_call.assert_has_calls([mock.call(['systemctl', @@ -465,13 +465,13 @@ class UtilsTestCase(unittest.TestCase): mock.call('json_testkey', 'neutron-api/0', 'hacluster:1'), ]) - @mock.patch.object(utils, 'render_template') + @mock.patch.object(utils, 'render') @mock.patch.object(utils.os.path, 'isdir') @mock.patch.object(utils.os, 'mkdir') @mock.patch.object(utils, 'write_file') @mock.patch.object(utils, 'config') def test_emit_base_conf(self, config, write_file, mkdir, isdir, - render_template): + mock_render): cfg = { 'corosync_key': 'Y29yb3N5bmNrZXkK', 'pacemaker_key': 'cGFjZW1ha2Vya2V5Cg==', @@ -482,7 +482,7 @@ class UtilsTestCase(unittest.TestCase): 'corosync': 'corosync etc default config', 'hacluster.acl': 'hacluster acl file', } - render_template.side_effect = lambda x, y: render[x] + mock_render.side_effect = lambda x, y: render[x] expect_write_calls = [ mock.call( content='corosync etc default config', @@ -515,16 +515,16 @@ class UtilsTestCase(unittest.TestCase): ] self.assertTrue(utils.emit_base_conf()) write_file.assert_has_calls(expect_write_calls) - render_template.assert_has_calls(expect_render_calls) + mock_render.assert_has_calls(expect_render_calls) mkdir.assert_has_calls(mkdir_calls) - @mock.patch.object(utils, 'render_template') + @mock.patch.object(utils, 'render') @mock.patch.object(utils.os.path, 'isdir') @mock.patch.object(utils.os, 'mkdir') @mock.patch.object(utils, 'write_file') @mock.patch.object(utils, 'config') def test_emit_base_conf_no_pcmkr_key(self, config, write_file, mkdir, - isdir, render_template): + isdir, mock_render): cfg = { 'corosync_key': 'Y29yb3N5bmNrZXkK', } @@ -534,7 +534,7 @@ class UtilsTestCase(unittest.TestCase): 'corosync': 'corosync etc default config', 'hacluster.acl': 'hacluster acl file', } - render_template.side_effect = lambda x, y: render[x] + mock_render.side_effect = lambda x, y: render[x] expect_write_calls = [ mock.call( content='corosync etc default config', @@ -567,16 +567,16 @@ class UtilsTestCase(unittest.TestCase): ] self.assertTrue(utils.emit_base_conf()) write_file.assert_has_calls(expect_write_calls) - render_template.assert_has_calls(expect_render_calls) + mock_render.assert_has_calls(expect_render_calls) mkdir.assert_has_calls(mkdir_calls) - @mock.patch.object(utils, 'render_template') + @mock.patch.object(utils, 'render') @mock.patch.object(utils.os.path, 'isdir') @mock.patch.object(utils.os, 'mkdir') @mock.patch.object(utils, 'write_file') @mock.patch.object(utils, 'config') def test_emit_base_conf_no_coro_key(self, config, write_file, mkdir, - isdir, render_template): + isdir, mock_render): cfg = { } config.side_effect = lambda x: cfg.get(x) @@ -585,7 +585,7 @@ class UtilsTestCase(unittest.TestCase): 'corosync': 'corosync etc default config', 'hacluster.acl': 'hacluster acl file', } - render_template.side_effect = lambda x, y: render[x] + mock_render.side_effect = lambda x, y: render[x] expect_write_calls = [ mock.call( content='corosync etc default config', @@ -608,7 +608,7 @@ class UtilsTestCase(unittest.TestCase): ] self.assertFalse(utils.emit_base_conf()) write_file.assert_has_calls(expect_write_calls) - render_template.assert_has_calls(expect_render_calls) + mock_render.assert_has_calls(expect_render_calls) mkdir.assert_has_calls(mkdir_calls) @mock.patch.object(utils, 'relation_get')