openstack
/
tripleo-common
Code Issues Proposed changes

Add support for jinja2 includes 

This add a custom jinja2 loader to fetch included templates from swift.

Change-Id: Idc5c3f49c7a2fc7f3622c76da001992cc657384e
Partially-Implements: blueprint overcloud-upgrades-per-service
This commit is contained in:
Oliver Walsh 2017-01-25 14:53:49 +00:00
parent 6f89d59517
commit 5935079b42
3 changed files with 177 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
releasenotes/notes/jinja2_include-8bef46285f25ddea.yaml Normal file
View File

@ -0,0 +1,3 @@
---
features:
- Adds support for the Jinja2 include statement in tripleo-heat-templates.

42
tripleo_common/actions/templates.py
View File

@ -41,6 +41,41 @@ def _create_temp_file(data):
return env_temp_file
class J2SwiftLoader(jinja2.BaseLoader):
"""Jinja2 loader to fetch included files from swift
This attempts to fetch a template include file from the given container.
An optional search path or list of search paths can be provided. By default
only the absolute path relative to the container root is searched.
"""
def __init__(self, swift, container, searchpath=None):
self.swift = swift
self.container = container
if searchpath is not None:
if isinstance(searchpath, six.string_types):
self.searchpath = [searchpath]
else:
self.searchpath = list(searchpath)
else:
self.searchpath = []
# Always search the absolute path from the root of the swift container
if '' not in self.searchpath:
self.searchpath.append('')
def get_source(self, environment, template):
pieces = jinja2.loaders.split_template_path(template)
for searchpath in self.searchpath:
template_path = os.path.join(searchpath, *pieces)
try:
source = self.swift.get_object(
self.container, template_path)[1]
return source, None, False
except swiftexceptions.ClientException:
pass
raise jinja2.exceptions.TemplateNotFound(template)
class UploadTemplatesAction(base.TripleOAction):
"""Upload default heat templates for TripleO."""
def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
@ -72,9 +107,14 @@ class ProcessTemplatesAction(base.TripleOAction):
swift = self.get_object_client()
yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
# Search for templates relative to the current template path first
template_base = os.path.dirname(yaml_f)
j2_loader = J2SwiftLoader(swift, self.container, template_base)
try:
# Render the j2 template
template = jinja2.Environment().from_string(j2_template)
template = jinja2.Environment(loader=j2_loader).from_string(
j2_template)
r_template = template.render(**j2_data)
except jinja2.exceptions.TemplateError as ex:
error_msg = ("Error rendering template %s : %s"

143
tripleo_common/tests/actions/test_templates.py
View File

@ -12,6 +12,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import jinja2
import mock
from swiftclient import exceptions as swiftexceptions
@ -20,7 +21,7 @@ from tripleo_common.actions import templates
from tripleo_common import constants
from tripleo_common.tests import base
JINJA_SNIPPET = """
JINJA_SNIPPET = r"""
# Jinja loop for Role in role_data.yaml
{% for role in roles %}
# Resources generated for {{role.name}} Role
@ -34,12 +35,12 @@ JINJA_SNIPPET = """
DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
{% endfor %}"""
ROLE_DATA_YAML = """
ROLE_DATA_YAML = r"""
-
name: CustomRole
"""
EXPECTED_JINJA_RESULT = """
EXPECTED_JINJA_RESULT = r"""
# Jinja loop for Role in role_data.yaml
# Resources generated for CustomRole Role
@ -53,30 +54,30 @@ EXPECTED_JINJA_RESULT = """
DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
"""
JINJA_SNIPPET_CONFIG = """
JINJA_SNIPPET_CONFIG = r"""
outputs:
OS::stack_id:
description: The software config which runs puppet on the {{role}} role
value: {get_resource: {{role}}PuppetConfigImpl}"""
J2_EXCLUDES = """
J2_EXCLUDES = r"""
name:
- puppet/controller-role.yaml
"""
J2_EXCLUDES_EMPTY_LIST = """
J2_EXCLUDES_EMPTY_LIST = r"""
name:
"""
J2_EXCLUDES_EMPTY_FILE = """
J2_EXCLUDES_EMPTY_FILE = r"""
"""
ROLE_DATA_DISABLE_CONSTRAINTS_YAML = """
ROLE_DATA_DISABLE_CONSTRAINTS_YAML = r"""
- name: RoleWithDisableConstraints
disable_constraints: True
"""
JINJA_SNIPPET_DISABLE_CONSTRAINTS = """
JINJA_SNIPPET_DISABLE_CONSTRAINTS = r"""
{{role}}Image:
type: string
default: overcloud-full
@ -86,7 +87,7 @@ JINJA_SNIPPET_DISABLE_CONSTRAINTS = """
{% endif %}
"""
EXPECTED_JINJA_RESULT_DISABLE_CONSTRAINTS = """
EXPECTED_JINJA_RESULT_DISABLE_CONSTRAINTS = r"""
RoleWithDisableConstraintsImage:
type: string
default: overcloud-full
@ -114,6 +115,69 @@ class UploadTemplatesActionTest(base.TestCase):
mock_get_swift.return_value, 'test', 'tar-container')
class J2SwiftLoaderTest(base.TestCase):
@staticmethod
def _setup_swift():
def return_multiple_files(*args):
if args[1] == 'bar/foo.yaml':
return ['', 'I am foo']
else:
raise swiftexceptions.ClientException('not found')
swift = mock.MagicMock()
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
return swift
def test_include_absolute_path(self):
j2_loader = templates.J2SwiftLoader(self._setup_swift(), None)
template = jinja2.Environment(loader=j2_loader).from_string(
r'''
Included this:
{% include 'bar/foo.yaml' %}
''')
self.assertEqual(
template.render(),
'''
Included this:
I am foo
''')
def test_include_search_path(self):
j2_loader = templates.J2SwiftLoader(self._setup_swift(), None, 'bar')
template = jinja2.Environment(loader=j2_loader).from_string(
r'''
Included this:
{% include 'foo.yaml' %}
''')
self.assertEqual(
template.render(),
'''
Included this:
I am foo
''')
def test_include_not_found(self):
j2_loader = templates.J2SwiftLoader(self._setup_swift(), None)
template = jinja2.Environment(loader=j2_loader).from_string(
r'''
Included this:
{% include 'bar.yaml' %}
''')
self.assertRaises(
jinja2.exceptions.TemplateNotFound,
template.render)
def test_include_invalid_path(self):
j2_loader = templates.J2SwiftLoader(self._setup_swift(), 'bar')
template = jinja2.Environment(loader=j2_loader).from_string(
r'''
Included this:
{% include '../foo.yaml' %}
''')
self.assertRaises(
jinja2.exceptions.TemplateNotFound,
template.render)
class ProcessTemplatesActionTest(base.TestCase):
@mock.patch('heatclient.common.template_utils.'
@ -269,6 +333,65 @@ class ProcessTemplatesActionTest(base.TestCase):
self.assertTrue("CustomRole" in str(action_result))
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('mistral.context.ctx')
def test_j2_render_and_put_include(self, ctx_mock, get_obj_client_mock):
def return_multiple_files(*args):
if args[1] == 'foo.yaml':
return ['', JINJA_SNIPPET_CONFIG]
def return_container_files(*args):
return ('headers', [{'name': 'foo.yaml'}])
# setup swift
swift = mock.MagicMock()
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
swift.get_container = mock.MagicMock(
side_effect=return_container_files)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
action._j2_render_and_put(r"{% include 'foo.yaml' %}",
{'role': 'CustomRole'},
'customrole-config.yaml')
action_result = swift.put_object._mock_mock_calls[0]
self.assertTrue("CustomRole" in str(action_result))
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('mistral.context.ctx')
def test_j2_render_and_put_include_relative(
self,
ctx_mock,
get_obj_client_mock):
def return_multiple_files(*args):
if args[1] == 'bar/foo.yaml':
return ['', JINJA_SNIPPET_CONFIG]
def return_container_files(*args):
return ('headers', [{'name': 'bar/foo.yaml'}])
# setup swift
swift = mock.MagicMock()
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
swift.get_container = mock.MagicMock(
side_effect=return_container_files)
get_obj_client_mock.return_value = swift
# Test
action = templates.ProcessTemplatesAction()
action._j2_render_and_put(r"{% include 'foo.yaml' %}",
{'role': 'CustomRole'},
'bar/customrole-config.yaml')
action_result = swift.put_object._mock_mock_calls[0]
self.assertTrue("CustomRole" in str(action_result))
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
@mock.patch('mistral.context.ctx')
def test_get_j2_excludes_file(self, ctx_mock, get_obj_client_mock):