Add local template generation tox task

This patch adds a local version of our template processing
routine so that developers can more quickly view the templates
that are actually getting generated. I've noticed multiple developers
now do a full deployment with 'overcloud deploy' only to download
the swift container with the generated templates. This simple task
avoids that step by allowing developers to generate it locally.

It also aims to preserve the ability to use t-h-t templates directly
with Heat (instead of going through Mistral) should users wish to do that.
The new undercloud heat installer requires the ability to generate
templates without requiring Mistral and Swift to do so.

Ideally the Mistral API workflow would use this same code
so perhaps in the future we might modify that routine to:

 -download swift tarball containing the templates
 -run this local routine that lives in t-h-t
 -re-upload the tarball of templates to the swift container

Change-Id: Ie664c9c5f455b7320a58a26f35bc403355408d9b
This commit is contained in:
Dan Prince 2016-11-07 07:45:15 -05:00
parent 6df32707e9
commit 4b5b24462b
5 changed files with 150 additions and 0 deletions

15
.gitignore vendored
View File

@ -44,3 +44,18 @@ doc/_build
# Built by pbr (python setup.py sdist):
AUTHORS
ChangeLog
extraconfig/all_nodes/mac_hostname.yaml
extraconfig/all_nodes/random_string.yaml
extraconfig/all_nodes/swap-partition.yaml
extraconfig/all_nodes/swap.yaml
extraconfig/tasks/major_upgrade_pacemaker_init.yaml
network/service_net_map.yaml
overcloud-resource-registry-puppet.yaml
overcloud.yaml
puppet/blockstorage-config.yaml
puppet/cephstorage-config.yaml
puppet/compute-config.yaml
puppet/controller-config.yaml
puppet/objectstorage-config.yaml
puppet/post.yaml

View File

@ -5,13 +5,17 @@ parameters:
description: Flavor for the {{role}} node.
default: baremetal
type: string
{% if disable_constraints is not defined %}
constraints:
- custom_constraint: nova.flavor
{% endif %}
{{role}}Image:
type: string
default: overcloud-full
{% if disable_constraints is not defined %}
constraints:
- custom_constraint: glance.image
{% endif %}
ImageUpdatePolicy:
default: 'REBUILD_PRESERVE_EPHEMERAL'
description: What policy to use when reconstructing instances. REBUILD for rebuilds, REBUILD_PRESERVE_EPHEMERAL to preserve /mnt.
@ -20,8 +24,10 @@ parameters:
description: Name of an existing Nova key pair to enable SSH access to the instances
type: string
default: default
{% if disable_constraints is not defined %}
constraints:
- custom_constraint: nova.keypair
{% endif %}
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set

View File

@ -1 +1,2 @@
pbr>=0.5.21,<1.0
Jinja2>=2.8 # BSD License (3 clause)

125
tools/process-templates.py Executable file
View File

@ -0,0 +1,125 @@
#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import jinja2
import os
import sys
import yaml
def parse_opts(argv):
parser = argparse.ArgumentParser(
description='Configure host network interfaces using a JSON'
' config file format.')
parser.add_argument('-p', '--base_path', metavar='BASE_PATH',
help="""base path of templates to process.""",
default='.')
parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA',
help="""relative path to the roles_data.yaml file.""",
default='roles_data.yaml')
parser.add_argument('--safe',
action='store_true',
help="""Enable safe mode (do not overwrite files).""",
default=False)
opts = parser.parse_args(argv[1:])
return opts
def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
overwrite=True):
yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
print('rendering j2 template to file: %s' % outfile_name)
if not overwrite and os.path.exists(outfile_name):
print('ERROR: path already exists for file: %s' % outfile_name)
sys.exit(1)
try:
# Render the j2 template
template = jinja2.Environment().from_string(j2_template)
r_template = template.render(**j2_data)
except jinja2.exceptions.TemplateError as ex:
error_msg = ("Error rendering template %s : %s"
% (yaml_f, six.text_type(ex)))
print(error_msg)
raise Exception(error_msg)
with open(outfile_name, 'w') as out_f:
out_f.write(r_template)
def process_templates(template_path, role_data_path, overwrite):
with open(role_data_path) as role_data_file:
role_data = yaml.safe_load(role_data_file)
j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml')
with open(j2_excludes_path) as role_data_file:
j2_excludes = yaml.safe_load(role_data_file)
role_names = [r.get('name') for r in role_data]
r_map = {}
for r in role_data:
r_map[r.get('name')] = r
excl_templates = ['%s/%s' % (template_path, e)
for e in j2_excludes.get('name')]
if os.path.isdir(template_path):
for subdir, dirs, files in os.walk(template_path):
for f in files:
file_path = os.path.join(subdir, f)
# We do two templating passes here:
# 1. *.role.j2.yaml - we template just the role name
# and create multiple files (one per role)
# 2. *.j2.yaml - we template with all roles_data,
# and create one file common to all roles
if f.endswith('.role.j2.yaml'):
print("jinja2 rendering role template %s" % f)
with open(file_path) as j2_template:
template_data = j2_template.read()
print("jinja2 rendering roles %s" % ","
.join(role_names))
for role in role_names:
j2_data = {'role': role}
# (dprince) For the undercloud installer we don't
# want to have heat check nova/glance API's
if r_map[role].get('disable_constraints', False):
j2_data['disable_constraints'] = True
out_f = "-".join(
[role.lower(),
os.path.basename(f).replace('.role.j2.yaml',
'.yaml')])
out_f_path = os.path.join(subdir, out_f)
if not (out_f_path in excl_templates):
_j2_render_to_file(template_data, j2_data,
out_f_path, overwrite)
else:
print('skipping rendering of %s' % out_f_path)
elif f.endswith('.j2.yaml'):
print("jinja2 rendering normal template %s" % f)
with open(file_path) as j2_template:
template_data = j2_template.read()
j2_data = {'roles': role_data}
out_f = file_path.replace('.j2.yaml', '.yaml')
_j2_render_to_file(template_data, j2_data, out_f,
overwrite)
else:
print('Unexpected argument %s' % template_path)
opts = parse_opts(sys.argv)
role_data_path = os.path.join(opts.base_path, opts.roles_data)
process_templates(opts.base_path, role_data_path, (not opts.safe))

View File

@ -12,3 +12,6 @@ commands = {posargs}
[testenv:pep8]
commands = python ./tools/yaml-validate.py .
[testenv:templates]
commands = python ./tools/process-templates.py .