Allow the usage of jinja2 templates

This drops the hard requirement on Cheetah.

Jinja is a python 2.4->3.x compatible templating engine, allow its
optional usage (until we can depreciate cheetah) by allowing for
specifying a template file header that can define which template engine to
use.

If the template file header does not specify a renderer, then assume
that that is cheetah.  If cheetah is not available, then use a limited
builtin renderer on a best effort basis, and log the warning.
This commit is contained in:
Scott Moser 2014-07-21 14:54:06 -04:00
commit aeca294990
13 changed files with 364 additions and 146 deletions

View File

@ -7,6 +7,8 @@
- doc: fix user-groups doc to reference plural ssh-authorized-keys
(LP: #1327065) [Joern Heissler]
- fix 'make test' in python 2.6
- support jinja2 as a templating engine. Drop the hard requirement on
cheetah. This helps in python3 effort. (LP: #1219223)
0.7.5:
- open 0.7.5
- Add a debug log message around import failures

View File

@ -20,13 +20,119 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from Cheetah.Template import Template
import collections
import re
try:
from Cheetah.Template import Template as CTemplate
CHEETAH_AVAILABLE = True
except (ImportError, AttributeError):
CHEETAH_AVAILABLE = False
try:
import jinja2
from jinja2 import Template as JTemplate
JINJA_AVAILABLE = True
except (ImportError, AttributeError):
JINJA_AVAILABLE = False
from cloudinit import log as logging
from cloudinit import type_utils as tu
from cloudinit import util
LOG = logging.getLogger(__name__)
TYPE_MATCHER = re.compile(r"##\s*template:(.*)", re.I)
BASIC_MATCHER = re.compile(r'\$\{([A-Za-z0-9_.]+)\}|\$([A-Za-z0-9_.]+)')
def basic_render(content, params):
"""This does simple replacement of bash variable like templates.
It identifies patterns like ${a} or $a and can also identify patterns like
${a.b} or $a.b which will look for a key 'b' in the dictionary rooted
by key 'a'.
"""
def replacer(match):
# Only 1 of the 2 groups will actually have a valid entry.
name = match.group(1)
if name is None:
name = match.group(2)
if name is None:
raise RuntimeError("Match encountered but no valid group present")
path = collections.deque(name.split("."))
selected_params = params
while len(path) > 1:
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError("Can not traverse into"
" non-dictionary '%s' of type %s while"
" looking for subkey '%s'"
% (selected_params,
tu.obj_name(selected_params),
key))
selected_params = selected_params[key]
key = path.popleft()
if not isinstance(selected_params, dict):
raise TypeError("Can not extract key '%s' from non-dictionary"
" '%s' of type %s"
% (key, selected_params,
tu.obj_name(selected_params)))
return str(selected_params[key])
return BASIC_MATCHER.sub(replacer, content)
def detect_template(text):
def cheetah_render(content, params):
return CTemplate(content, searchList=[params]).respond()
def jinja_render(content, params):
return JTemplate(content,
undefined=jinja2.StrictUndefined,
trim_blocks=True).render(**params)
if text.find("\n") != -1:
ident, rest = text.split("\n", 1)
else:
ident = text
rest = ''
type_match = TYPE_MATCHER.match(ident)
if not type_match:
if not CHEETAH_AVAILABLE:
LOG.warn("Cheetah not available as the default renderer for"
" unknown template, reverting to the basic renderer.")
return ('basic', basic_render, text)
else:
return ('cheetah', cheetah_render, text)
else:
template_type = type_match.group(1).lower().strip()
if template_type not in ('jinja', 'cheetah', 'basic'):
raise ValueError("Unknown template rendering type '%s' requested"
% template_type)
if template_type == 'jinja' and not JINJA_AVAILABLE:
LOG.warn("Jinja not available as the selected renderer for"
" desired template, reverting to the basic renderer.")
return ('basic', basic_render, rest)
elif template_type == 'jinja' and JINJA_AVAILABLE:
return ('jinja', jinja_render, rest)
if template_type == 'cheetah' and not CHEETAH_AVAILABLE:
LOG.warn("Cheetah not available as the selected renderer for"
" desired template, reverting to the basic renderer.")
return ('basic', basic_render, rest)
elif template_type == 'cheetah' and CHEETAH_AVAILABLE:
return ('cheetah', cheetah_render, rest)
# Only thing left over is the basic renderer (it is always available).
return ('basic', basic_render, rest)
def render_from_file(fn, params):
return render_string(util.load_file(fn), params)
if not params:
params = {}
template_type, renderer, content = detect_template(util.load_file(fn))
LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type)
return renderer(content, params)
def render_to_file(fn, outfn, params, mode=0644):
@ -37,4 +143,5 @@ def render_to_file(fn, outfn, params, mode=0644):
def render_string(content, params):
if not params:
params = {}
return Template(content, searchList=[params]).respond()
template_type, renderer, content = detect_template(content)
return renderer(content, params)

View File

@ -31,6 +31,7 @@ PKG_MP = {
'argparse': 'python-argparse',
'cheetah': 'python-cheetah',
'configobj': 'python-configobj',
'jinja2': 'python-jinja2',
'jsonpatch': 'python-jsonpatch | python-json-patch',
'oauth': 'python-oauth',
'prettytable': 'python-prettytable',

View File

@ -37,6 +37,7 @@ PKG_MP = {
'redhat': {
'argparse': 'python-argparse',
'cheetah': 'python-cheetah',
'jinja2': 'python-jinja2',
'configobj': 'python-configobj',
'jsonpatch': 'python-jsonpatch',
'oauth': 'python-oauth',

View File

@ -2,6 +2,7 @@
# Used for untemplating any files or strings with parameters.
cheetah
jinja2
# This is used for any pretty printing of tabular data.
PrettyTable

View File

@ -1,25 +1,25 @@
#*
This file is only utilized if the module 'cc_chef' is enabled in
cloud-config. Specifically, in order to enable it
you need to add the following to config:
chef:
validation_key: XYZ
validation_cert: XYZ
validation_name: XYZ
server_url: XYZ
*#
## template:jinja
{#
This file is only utilized if the module 'cc_chef' is enabled in
cloud-config. Specifically, in order to enable it
you need to add the following to config:
chef:
validation_key: XYZ
validation_cert: XYZ
validation_name: XYZ
server_url: XYZ
-#}
log_level :info
log_location "/var/log/chef/client.log"
ssl_verify_mode :verify_none
validation_client_name "$validation_name"
validation_client_name "{{validation_name}}"
validation_key "/etc/chef/validation.pem"
client_key "/etc/chef/client.pem"
chef_server_url "$server_url"
environment "$environment"
node_name "$node_name"
chef_server_url "{{server_url}}"
environment "{{environment}}"
node_name "{{node_name}}"
json_attribs "/etc/chef/firstboot.json"
file_cache_path "/var/cache/chef"
file_backup_path "/var/backups/chef"
pid_file "/var/run/chef/client.pid"
Chef::Log::Formatter.show_time = true

View File

@ -1,19 +1,19 @@
## This file (/etc/cloud/templates/hosts.tmpl) is only utilized
## if enabled in cloud-config. Specifically, in order to enable it
## you need to add the following to config:
## manage_etc_hosts: True
##
## Note, double-hash commented lines will not appear in /etc/hosts
#
## template:jinja
{#
This file (/etc/cloud/templates/hosts.tmpl) is only utilized
if enabled in cloud-config. Specifically, in order to enable it
you need to add the following to config:
manage_etc_hosts: True
-#}
# Your system has configured 'manage_etc_hosts' as True.
# As a result, if you wish for changes to this file to persist
# then you will need to either
# a.) make changes to the master file in /etc/cloud/templates/hosts.tmpl
# b.) change or remove the value of 'manage_etc_hosts' in
# /etc/cloud/cloud.cfg or cloud-config from user-data
#
## The value '$hostname' will be replaced with the local-hostname
127.0.1.1 $fqdn $hostname
#
{# The value '{{hostname}}' will be replaced with the local-hostname -#}
127.0.1.1 {{fqdn}} {{hostname}}
127.0.0.1 localhost
# The following lines are desirable for IPv6 capable hosts
@ -23,3 +23,4 @@ ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts

View File

@ -1,9 +1,10 @@
#*
This file /etc/cloud/templates/hosts.redhat.tmpl is only utilized
if enabled in cloud-config. Specifically, in order to enable it
you need to add the following to config:
manage_etc_hosts: True
*#
## template:jinja
{#
This file /etc/cloud/templates/hosts.redhat.tmpl is only utilized
if enabled in cloud-config. Specifically, in order to enable it
you need to add the following to config:
manage_etc_hosts: True
-#}
# Your system has configured 'manage_etc_hosts' as True.
# As a result, if you wish for changes to this file to persist
# then you will need to either
@ -12,12 +13,12 @@
# /etc/cloud/cloud.cfg or cloud-config from user-data
#
# The following lines are desirable for IPv4 capable hosts
127.0.0.1 ${fqdn} ${hostname}
127.0.0.1 {{fqdn}} {{hostname}}
127.0.0.1 localhost.localdomain localhost
127.0.0.1 localhost4.localdomain4 localhost4
# The following lines are desirable for IPv6 capable hosts
::1 ${fqdn} ${hostname}
::1 {{fqdn}} {{hostname}}
::1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6

View File

@ -1,9 +1,10 @@
#*
This file /etc/cloud/templates/hosts.suse.tmpl is only utilized
if enabled in cloud-config. Specifically, in order to enable it
you need to add the following to config:
manage_etc_hosts: True
*#
## template:jinja
{#
This file /etc/cloud/templates/hosts.suse.tmpl is only utilized
if enabled in cloud-config. Specifically, in order to enable it
you need to add the following to config:
manage_etc_hosts: True
-#}
# Your system has configured 'manage_etc_hosts' as True.
# As a result, if you wish for changes to this file to persist
# then you will need to either
@ -22,3 +23,4 @@ ff00::0 ipv6-mcastprefix
ff02::1 ipv6-allnodes
ff02::2 ipv6-allrouters
ff02::3 ipv6-allhosts

View File

@ -1,39 +1,30 @@
#
## template:jinja
# Your system has been configured with 'manage-resolv-conf' set to true.
# As a result, cloud-init has written this file with configuration data
# that it has been provided. Cloud-init, by default, will write this file
# a single time (PER_ONCE).
#
{% if nameservers is defined %}
{% for server in nameservers %}
nameserver {{server}}
{% endfor %}
#if $varExists('nameservers')
#for $server in $nameservers
nameserver $server
#end for
#end if
#if $varExists('searchdomains')
search #slurp
#for $search in $searchdomains
$search #slurp
#end for
{% endif -%}
{% if searchdomains is defined %}
search {% for search in searchdomains %}{{search}} {% endfor %}
#end if
#if $varExists('domain')
domain $domain
#end if
#if $varExists('sortlist')
sortlist #slurp
#for $sort in $sortlist
$sort #slurp
#end for
{% endif %}
{% if domain is defined %}
domain {{domain}}
{% endif %}
{% if sortlist is defined %}
#end if
#if $varExists('options') or $varExists('flags')
options #slurp
#for $flag in $flags
$flag #slurp
#end for
#for $key, $value in $options.items()
$key:$value #slurp
#end for
sortlist {% for sort in sortlist %}{{sort}} {% endfor %}
{% endif %}
{% if options is defined or flags is defined %}
#end if
options {% for flag in flags %}{{flag}} {% endfor %}
{% for key, value in options.iteritems() -%}
{{key}}:{{value}}
{% endfor %}
{% endif %}

View File

@ -1,28 +1,32 @@
\## Note, this file is written by cloud-init on first boot of an instance
\## modifications made here will not survive a re-bundle.
\## if you wish to make changes you can:
\## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
\## or do the same in user-data
\## b.) add sources in /etc/apt/sources.list.d
\## c.) make changes to template file /etc/cloud/templates/sources.list.debian.tmpl
\###
## template:jinja
## Note, this file is written by cloud-init on first boot of an instance
## modifications made here will not survive a re-bundle.
## if you wish to make changes you can:
## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
## or do the same in user-data
## b.) add sources in /etc/apt/sources.list.d
## c.) make changes to template file /etc/cloud/templates/sources.list.debian.tmpl
###
# See http://www.debian.org/releases/stable/i386/release-notes/ch-upgrading.html
# for how to upgrade to newer versions of the distribution.
deb $mirror $codename main contrib non-free
deb-src $mirror $codename main contrib non-free
deb {{mirror}} {{codename}} main contrib non-free
deb-src {{mirror}} {{codename}} main contrib non-free
\## Major bug fix updates produced after the final release of the
\## distribution.
deb $security $codename/updates main contrib non-free
deb-src $security $codename/updates main contrib non-free
deb $mirror $codename-updates main contrib non-free
deb-src $mirror $codename-updates main contrib non-free
## Major bug fix updates produced after the final release of the
## distribution.
deb {{security}} {{codename}}/updates main contrib non-free
deb-src {{security}} {{codename}}/updates main contrib non-free
deb {{mirror}} {{codename}}-updates main contrib non-free
deb-src {{mirror}} {{codename}}-updates main contrib non-free
\## Uncomment the following two lines to add software from the 'backports'
\## repository.
\## N.B. software from this repository may not have been tested as
\## extensively as that contained in the main release, although it includes
\## newer versions of some applications which may provide useful features.
# deb http://backports.debian.org/debian-backports $codename-backports main contrib non-free
# deb-src http://backports.debian.org/debian-backports $codename-backports main contrib non-free
## Uncomment the following two lines to add software from the 'backports'
## repository.
##
## N.B. software from this repository may not have been tested as
## extensively as that contained in the main release, although it includes
## newer versions of some applications which may provide useful features.
{#
deb http://backports.debian.org/debian-backports {{codename}}-backports main contrib non-free
deb-src http://backports.debian.org/debian-backports {{codename}}-backports main contrib non-free
-#}

View File

@ -1,60 +1,60 @@
\## Note, this file is written by cloud-init on first boot of an instance
\## modifications made here will not survive a re-bundle.
\## if you wish to make changes you can:
\## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
\## or do the same in user-data
\## b.) add sources in /etc/apt/sources.list.d
\## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
\###
## template:jinja
## Note, this file is written by cloud-init on first boot of an instance
## modifications made here will not survive a re-bundle.
## if you wish to make changes you can:
## a.) add 'apt_preserve_sources_list: true' to /etc/cloud/cloud.cfg
## or do the same in user-data
## b.) add sources in /etc/apt/sources.list.d
## c.) make changes to template file /etc/cloud/templates/sources.list.tmpl
# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
# newer versions of the distribution.
deb $mirror $codename main
deb-src $mirror $codename main
deb {{mirror}} {{codename}} main
deb-src {{mirror}} {{codename}} main
\## Major bug fix updates produced after the final release of the
\## distribution.
deb $mirror $codename-updates main
deb-src $mirror $codename-updates main
## Major bug fix updates produced after the final release of the
## distribution.
deb {{mirror}} {{codename}}-updates main
deb-src {{mirror}} {{codename}}-updates main
\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
\## team. Also, please note that software in universe WILL NOT receive any
\## review or updates from the Ubuntu security team.
deb $mirror $codename universe
deb-src $mirror $codename universe
deb $mirror $codename-updates universe
deb-src $mirror $codename-updates universe
## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team. Also, please note that software in universe WILL NOT receive any
## review or updates from the Ubuntu security team.
deb {{mirror}} {{codename}} universe
deb-src {{mirror}} {{codename}} universe
deb {{mirror}} {{codename}}-updates universe
deb-src {{mirror}} {{codename}}-updates universe
\## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
\## team, and may not be under a free licence. Please satisfy yourself as to
\## your rights to use the software. Also, please note that software in
\## multiverse WILL NOT receive any review or updates from the Ubuntu
\## security team.
# deb $mirror $codename multiverse
# deb-src $mirror $codename multiverse
# deb $mirror $codename-updates multiverse
# deb-src $mirror $codename-updates multiverse
## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team, and may not be under a free licence. Please satisfy yourself as to
## your rights to use the software. Also, please note that software in
## multiverse WILL NOT receive any review or updates from the Ubuntu
## security team.
# deb {{mirror}} {{codename}} multiverse
# deb-src {{mirror}} {{codename}} multiverse
# deb {{mirror}} {{codename}}-updates multiverse
# deb-src {{mirror}} {{codename}}-updates multiverse
\## Uncomment the following two lines to add software from the 'backports'
\## repository.
\## N.B. software from this repository may not have been tested as
\## extensively as that contained in the main release, although it includes
\## newer versions of some applications which may provide useful features.
\## Also, please note that software in backports WILL NOT receive any review
\## or updates from the Ubuntu security team.
# deb $mirror $codename-backports main restricted universe multiverse
# deb-src $mirror $codename-backports main restricted universe multiverse
## Uncomment the following two lines to add software from the 'backports'
## repository.
## N.B. software from this repository may not have been tested as
## extensively as that contained in the main release, although it includes
## newer versions of some applications which may provide useful features.
## Also, please note that software in backports WILL NOT receive any review
## or updates from the Ubuntu security team.
# deb {{mirror}} {{codename}}-backports main restricted universe multiverse
# deb-src {{mirror}} {{codename}}-backports main restricted universe multiverse
\## Uncomment the following two lines to add software from Canonical's
\## 'partner' repository.
\## This software is not part of Ubuntu, but is offered by Canonical and the
\## respective vendors as a service to Ubuntu users.
# deb http://archive.canonical.com/ubuntu $codename partner
# deb-src http://archive.canonical.com/ubuntu $codename partner
## Uncomment the following two lines to add software from Canonical's
## 'partner' repository.
## This software is not part of Ubuntu, but is offered by Canonical and the
## respective vendors as a service to Ubuntu users.
# deb http://archive.canonical.com/ubuntu {{codename}} partner
# deb-src http://archive.canonical.com/ubuntu {{codename}} partner
deb $security $codename-security main
deb-src $security $codename-security main
deb $security $codename-security universe
deb-src $security $codename-security universe
# deb $security $codename-security multiverse
# deb-src $security $codename-security multiverse
deb {{security}} {{codename}}-security main
deb-src {{security}} {{codename}}-security main
deb {{security}} {{codename}}-security universe
deb-src {{security}} {{codename}}-security universe
# deb {{security}} {{codename}}-security multiverse
# deb-src {{security}} {{codename}}-security multiverse

View File

@ -0,0 +1,107 @@
# vi: ts=4 expandtab
#
# Copyright (C) 2014 Yahoo! Inc.
#
# Author: Joshua Harlow <harlowja@yahoo-inc.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from tests.unittests import helpers as test_helpers
import textwrap
from cloudinit import templater
class TestTemplates(test_helpers.TestCase):
def test_render_basic(self):
in_data = textwrap.dedent("""
${b}
c = d
""")
in_data = in_data.strip()
expected_data = textwrap.dedent("""
2
c = d
""")
out_data = templater.basic_render(in_data, {'b': 2})
self.assertEqual(expected_data.strip(), out_data)
def test_detection(self):
blob = "## template:cheetah"
(template_type, renderer, contents) = templater.detect_template(blob)
self.assertIn("cheetah", template_type)
self.assertEqual("", contents.strip())
blob = "blahblah $blah"
(template_type, renderer, contents) = templater.detect_template(blob)
self.assertIn("cheetah", template_type)
self.assertEquals(blob, contents)
blob = '##template:something-new'
self.assertRaises(ValueError, templater.detect_template, blob)
def test_render_cheetah(self):
blob = '''## template:cheetah
$a,$b'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEquals("1,2", c)
def test_render_jinja(self):
blob = '''## template:jinja
{{a}},{{b}}'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEquals("1,2", c)
def test_render_default(self):
blob = '''$a,$b'''
c = templater.render_string(blob, {"a": 1, "b": 2})
self.assertEquals("1,2", c)
def test_render_basic_deeper(self):
hn = 'myfoohost.yahoo.com'
expected_data = "h=%s\nc=d\n" % hn
in_data = "h=$hostname.canonical_name\nc=d\n"
params = {
"hostname": {
"canonical_name": hn,
},
}
out_data = templater.render_string(in_data, params)
self.assertEqual(expected_data, out_data)
def test_render_basic_no_parens(self):
hn = "myfoohost"
in_data = "h=$hostname\nc=d\n"
expected_data = "h=%s\nc=d\n" % hn
out_data = templater.basic_render(in_data, {'hostname': hn})
self.assertEqual(expected_data, out_data)
def test_render_basic_parens(self):
hn = "myfoohost"
in_data = "h = ${hostname}\nc=d\n"
expected_data = "h = %s\nc=d\n" % hn
out_data = templater.basic_render(in_data, {'hostname': hn})
self.assertEqual(expected_data, out_data)
def test_render_basic2(self):
mirror = "mymirror"
codename = "zany"
in_data = "deb $mirror $codename-updates main contrib non-free"
ex_data = "deb %s %s-updates main contrib non-free" % (mirror, codename)
out_data = templater.basic_render(in_data,
{'mirror': mirror, 'codename': codename})
self.assertEqual(ex_data, out_data)