Switch to render from charmhelpers

- add contrib/templating
- using render instead of render_template
- remove render_template function

Change-Id: I395d7dc06618998b9e6023ff649f4aa8c5930cc0

charm-helpers-hooks.yaml

@@ -20,3 +20,4 @@ include:
     - contrib.python
     - contrib.charmsupport
     - contrib.openstack.policyd
+    - contrib.templating

charmhelpers/contrib/templating/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

charmhelpers/contrib/templating/contexts.py

@@ -0,0 +1,137 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2013 Canonical Ltd.
+#
+# Authors:
+#  Charm Helpers Developers <juju@lists.ubuntu.com>
+"""A helper to create a yaml cache of config with namespaced relation data."""
+import os
+import yaml
+
+import six
+
+import charmhelpers.core.hookenv
+
+
+charm_dir = os.environ.get('CHARM_DIR', '')
+
+
+def dict_keys_without_hyphens(a_dict):
+    """Return the a new dict with underscores instead of hyphens in keys."""
+    return dict(
+        (key.replace('-', '_'), val) for key, val in a_dict.items())
+
+
+def update_relations(context, namespace_separator=':'):
+    """Update the context with the relation data."""
+    # Add any relation data prefixed with the relation type.
+    relation_type = charmhelpers.core.hookenv.relation_type()
+    relations = []
+    context['current_relation'] = {}
+    if relation_type is not None:
+        relation_data = charmhelpers.core.hookenv.relation_get()
+        context['current_relation'] = relation_data
+        # Deprecated: the following use of relation data as keys
+        # directly in the context will be removed.
+        relation_data = dict(
+            ("{relation_type}{namespace_separator}{key}".format(
+                relation_type=relation_type,
+                key=key,
+                namespace_separator=namespace_separator), val)
+            for key, val in relation_data.items())
+        relation_data = dict_keys_without_hyphens(relation_data)
+        context.update(relation_data)
+        relations = charmhelpers.core.hookenv.relations_of_type(relation_type)
+        relations = [dict_keys_without_hyphens(rel) for rel in relations]
+
+    context['relations_full'] = charmhelpers.core.hookenv.relations()
+
+    # the hookenv.relations() data structure is effectively unusable in
+    # templates and other contexts when trying to access relation data other
+    # than the current relation. So provide a more useful structure that works
+    # with any hook.
+    local_unit = charmhelpers.core.hookenv.local_unit()
+    relations = {}
+    for rname, rids in context['relations_full'].items():
+        relations[rname] = []
+        for rid, rdata in rids.items():
+            data = rdata.copy()
+            if local_unit in rdata:
+                data.pop(local_unit)
+            for unit_name, rel_data in data.items():
+                new_data = {'__relid__': rid, '__unit__': unit_name}
+                new_data.update(rel_data)
+                relations[rname].append(new_data)
+    context['relations'] = relations
+
+
+def juju_state_to_yaml(yaml_path, namespace_separator=':',
+                       allow_hyphens_in_keys=True, mode=None):
+    """Update the juju config and state in a yaml file.
+
+    This includes any current relation-get data, and the charm
+    directory.
+
+    This function was created for the ansible and saltstack
+    support, as those libraries can use a yaml file to supply
+    context to templates, but it may be useful generally to
+    create and update an on-disk cache of all the config, including
+    previous relation data.
+
+    By default, hyphens are allowed in keys as this is supported
+    by yaml, but for tools like ansible, hyphens are not valid [1].
+
+    [1] http://www.ansibleworks.com/docs/playbooks_variables.html#what-makes-a-valid-variable-name
+    """
+    config = charmhelpers.core.hookenv.config()
+
+    # Add the charm_dir which we will need to refer to charm
+    # file resources etc.
+    config['charm_dir'] = charm_dir
+    config['local_unit'] = charmhelpers.core.hookenv.local_unit()
+    config['unit_private_address'] = charmhelpers.core.hookenv.unit_private_ip()
+    config['unit_public_address'] = charmhelpers.core.hookenv.unit_get(
+        'public-address'
+    )
+
+    # Don't use non-standard tags for unicode which will not
+    # work when salt uses yaml.load_safe.
+    yaml.add_representer(six.text_type,
+                         lambda dumper, value: dumper.represent_scalar(
+                             six.u('tag:yaml.org,2002:str'), value))
+
+    yaml_dir = os.path.dirname(yaml_path)
+    if not os.path.exists(yaml_dir):
+        os.makedirs(yaml_dir)
+
+    if os.path.exists(yaml_path):
+        with open(yaml_path, "r") as existing_vars_file:
+            existing_vars = yaml.load(existing_vars_file.read())
+    else:
+        with open(yaml_path, "w+"):
+            pass
+        existing_vars = {}
+
+    if mode is not None:
+        os.chmod(yaml_path, mode)
+
+    if not allow_hyphens_in_keys:
+        config = dict_keys_without_hyphens(config)
+    existing_vars.update(config)
+
+    update_relations(existing_vars, namespace_separator)
+
+    with open(yaml_path, "w+") as fp:
+        fp.write(yaml.dump(existing_vars, default_flow_style=False))

charmhelpers/contrib/templating/jinja.py

@@ -0,0 +1,51 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Templating using the python-jinja2 package.
+"""
+import six
+from charmhelpers.fetch import apt_install, apt_update
+try:
+    import jinja2
+except ImportError:
+    apt_update(fatal=True)
+    if six.PY3:
+        apt_install(["python3-jinja2"], fatal=True)
+    else:
+        apt_install(["python-jinja2"], fatal=True)
+    import jinja2
+
+
+DEFAULT_TEMPLATES_DIR = 'templates'
+
+
+def render(template_name, context, template_dir=DEFAULT_TEMPLATES_DIR,
+           jinja_env_args=None):
+    """
+    Render jinja2 template with provided context.
+
+    :param template_name: name of the jinja template file
+    :param context: template context
+    :param template_dir: directory in which the template file is located
+    :param jinja_env_args: additional arguments passed to the
+                           jinja2.Environment. Expected dict with format
+                           {'arg_name': 'arg_value'}
+    :return: Rendered template as a string
+    """
+    env_kwargs = jinja_env_args or {}
+    templates = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(template_dir), **env_kwargs)
+    template = templates.get_template(template_name)
+    return template.render(context)

charmhelpers/contrib/templating/pyformat.py

@@ -0,0 +1,27 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+Templating using standard Python str.format() method.
+'''
+
+from charmhelpers.core import hookenv
+
+
+def render(template, extra={}, **kwargs):
+    """Return the template rendered using Python's str.format()."""
+    context = hookenv.execution_environment()
+    context.update(extra)
+    context.update(kwargs)
+    return template.format(**context)

hooks/utils.py

@@ -61,6 +61,7 @@ from charmhelpers.contrib.openstack.utils import (
 from charmhelpers.contrib.openstack.ha.utils import (
     assert_charm_supports_dns_ha
 )
+from charmhelpers.contrib.templating.jinja import render
 from charmhelpers.core.host import (
     mkdir,
     rsync,
@@ -85,10 +86,8 @@ from charmhelpers.contrib.network import ip as utils
 
 import netifaces
 from netaddr import IPNetwork
-import jinja2
 
 
-TEMPLATES_DIR = 'templates'
 COROSYNC_CONF = '/etc/corosync/corosync.conf'
 COROSYNC_DEFAULT = '/etc/default/corosync'
 COROSYNC_AUTHKEY = '/etc/corosync/authkey'
@@ -346,8 +345,8 @@ def emit_systemd_overrides_file():
             os.mkdir(overrides_dir)
 
         write_file(path=overrides_file,
-                   content=render_template('systemd-overrides.conf',
-                                           systemd_overrides_context))
+                   content=render('systemd-overrides.conf',
+                                  systemd_overrides_context))
 
     # Update systemd with the new information
     subprocess.check_call(['systemctl', 'daemon-reload'])
@@ -357,8 +356,7 @@ def emit_corosync_conf():
     corosync_conf_context = get_corosync_conf()
     if corosync_conf_context:
         write_file(path=COROSYNC_CONF,
-                   content=render_template('corosync.conf',
-                                           corosync_conf_context))
+                   content=render('corosync.conf', corosync_conf_context))
         return True
 
     return False
@@ -376,11 +374,10 @@ def emit_base_conf():
         os.mkdir(PCMKR_CONFIG_DIR)
     corosync_default_context = {'corosync_enabled': 'yes'}
     write_file(path=COROSYNC_DEFAULT,
-               content=render_template('corosync',
-                                       corosync_default_context))
+               content=render('corosync', corosync_default_context))
 
     write_file(path=COROSYNC_HACLUSTER_ACL,
-               content=render_template('hacluster.acl', {}))
+               content=render('hacluster.acl', {}))
 
     corosync_key = config('corosync_key')
     if corosync_key:
@@ -398,14 +395,6 @@ def emit_base_conf():
     return False
 
 
-def render_template(template_name, context, template_dir=TEMPLATES_DIR):
-    templates = jinja2.Environment(
-        loader=jinja2.FileSystemLoader(template_dir)
-    )
-    template = templates.get_template(template_name)
-    return template.render(context)
-
-
 def assert_charm_supports_ipv6():
     """Check whether we are able to support charms ipv6."""
     _release = lsb_release()['DISTRIB_CODENAME'].lower()

test-requirements.txt

@@ -37,7 +37,8 @@ git+https://github.com/openstack-charmers/zaza.git#egg=zaza
 git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
 
 # Needed for charm-glance:
-git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.6'
+git+https://opendev.org/openstack/tempest.git#egg=tempest;python_version>='3.8'
+tempest<31.0.0;python_version<'3.8'
 tempest<24.0.0;python_version<'3.6'
 
 croniter            # needed for charm-rabbitmq-server unit tests

unit_tests/test_hacluster_utils.py

@@ -320,13 +320,13 @@ class UtilsTestCase(unittest.TestCase):
     @mock.patch.object(subprocess, 'check_call')
     @mock.patch.object(utils.os, 'mkdir')
     @mock.patch.object(utils.os.path, 'exists')
-    @mock.patch.object(utils, 'render_template')
+    @mock.patch.object(utils, 'render')
     @mock.patch.object(utils, 'write_file')
     @mock.patch.object(utils, 'is_unit_paused_set')
     @mock.patch.object(utils, 'config')
     def test_emit_systemd_overrides_file(self, mock_config,
                                          mock_is_unit_paused_set,
-                                         mock_write_file, mock_render_template,
+                                         mock_write_file, mock_render,
                                          mock_path_exists,
                                          mock_mkdir, mock_check_call):
 
@@ -339,13 +339,13 @@ class UtilsTestCase(unittest.TestCase):
         mock_path_exists.return_value = True
         utils.emit_systemd_overrides_file()
         self.assertEqual(2, len(mock_write_file.mock_calls))
-        mock_render_template.assert_has_calls(
+        mock_render.assert_has_calls(
             [mock.call('systemd-overrides.conf', cfg),
              mock.call('systemd-overrides.conf', cfg)])
         mock_check_call.assert_has_calls([mock.call(['systemctl',
                                                      'daemon-reload'])])
         mock_write_file.mock_calls = []
-        mock_render_template.mock_calls = []
+        mock_render.mock_calls = []
         mock_check_call.mock_calls = []
 
         # Disable timeout
@@ -358,7 +358,7 @@ class UtilsTestCase(unittest.TestCase):
         mock_path_exists.return_value = True
         utils.emit_systemd_overrides_file()
         self.assertEqual(2, len(mock_write_file.mock_calls))
-        mock_render_template.assert_has_calls(
+        mock_render.assert_has_calls(
             [mock.call('systemd-overrides.conf', expected_cfg),
              mock.call('systemd-overrides.conf', expected_cfg)])
         mock_check_call.assert_has_calls([mock.call(['systemctl',
@@ -465,13 +465,13 @@ class UtilsTestCase(unittest.TestCase):
             mock.call('json_testkey', 'neutron-api/0', 'hacluster:1'),
         ])
 
-    @mock.patch.object(utils, 'render_template')
+    @mock.patch.object(utils, 'render')
     @mock.patch.object(utils.os.path, 'isdir')
     @mock.patch.object(utils.os, 'mkdir')
     @mock.patch.object(utils, 'write_file')
     @mock.patch.object(utils, 'config')
     def test_emit_base_conf(self, config, write_file, mkdir, isdir,
-                            render_template):
+                            mock_render):
         cfg = {
             'corosync_key': 'Y29yb3N5bmNrZXkK',
             'pacemaker_key': 'cGFjZW1ha2Vya2V5Cg==',
@@ -482,7 +482,7 @@ class UtilsTestCase(unittest.TestCase):
             'corosync': 'corosync etc default config',
             'hacluster.acl': 'hacluster acl file',
         }
-        render_template.side_effect = lambda x, y: render[x]
+        mock_render.side_effect = lambda x, y: render[x]
         expect_write_calls = [
             mock.call(
                 content='corosync etc default config',
@@ -515,16 +515,16 @@ class UtilsTestCase(unittest.TestCase):
         ]
         self.assertTrue(utils.emit_base_conf())
         write_file.assert_has_calls(expect_write_calls)
-        render_template.assert_has_calls(expect_render_calls)
+        mock_render.assert_has_calls(expect_render_calls)
         mkdir.assert_has_calls(mkdir_calls)
 
-    @mock.patch.object(utils, 'render_template')
+    @mock.patch.object(utils, 'render')
     @mock.patch.object(utils.os.path, 'isdir')
     @mock.patch.object(utils.os, 'mkdir')
     @mock.patch.object(utils, 'write_file')
     @mock.patch.object(utils, 'config')
     def test_emit_base_conf_no_pcmkr_key(self, config, write_file, mkdir,
-                                         isdir, render_template):
+                                         isdir, mock_render):
         cfg = {
             'corosync_key': 'Y29yb3N5bmNrZXkK',
         }
@@ -534,7 +534,7 @@ class UtilsTestCase(unittest.TestCase):
             'corosync': 'corosync etc default config',
             'hacluster.acl': 'hacluster acl file',
         }
-        render_template.side_effect = lambda x, y: render[x]
+        mock_render.side_effect = lambda x, y: render[x]
         expect_write_calls = [
             mock.call(
                 content='corosync etc default config',
@@ -567,16 +567,16 @@ class UtilsTestCase(unittest.TestCase):
         ]
         self.assertTrue(utils.emit_base_conf())
         write_file.assert_has_calls(expect_write_calls)
-        render_template.assert_has_calls(expect_render_calls)
+        mock_render.assert_has_calls(expect_render_calls)
         mkdir.assert_has_calls(mkdir_calls)
 
-    @mock.patch.object(utils, 'render_template')
+    @mock.patch.object(utils, 'render')
     @mock.patch.object(utils.os.path, 'isdir')
     @mock.patch.object(utils.os, 'mkdir')
     @mock.patch.object(utils, 'write_file')
     @mock.patch.object(utils, 'config')
     def test_emit_base_conf_no_coro_key(self, config, write_file, mkdir,
-                                        isdir, render_template):
+                                        isdir, mock_render):
         cfg = {
         }
         config.side_effect = lambda x: cfg.get(x)
@@ -585,7 +585,7 @@ class UtilsTestCase(unittest.TestCase):
             'corosync': 'corosync etc default config',
             'hacluster.acl': 'hacluster acl file',
         }
-        render_template.side_effect = lambda x, y: render[x]
+        mock_render.side_effect = lambda x, y: render[x]
         expect_write_calls = [
             mock.call(
                 content='corosync etc default config',
@@ -608,7 +608,7 @@ class UtilsTestCase(unittest.TestCase):
         ]
         self.assertFalse(utils.emit_base_conf())
         write_file.assert_has_calls(expect_write_calls)
-        render_template.assert_has_calls(expect_render_calls)
+        mock_render.assert_has_calls(expect_render_calls)
         mkdir.assert_has_calls(mkdir_calls)
 
     @mock.patch.object(utils, 'relation_get')