Move process template functionality to utils
Change-Id: I6c48d8dd3c5c480eef0001746ce5e243d6820be9
This commit is contained in:
parent
dc3a909f7d
commit
4d969f0456
|
@ -126,7 +126,6 @@ mistral.actions =
|
|||
tripleo.scale.delete_node = tripleo_common.actions.scale:ScaleDownAction
|
||||
tripleo.swift.tempurl = tripleo_common.actions.swifthelper:SwiftTempUrlAction
|
||||
tripleo.swift.swift_information = tripleo_common.actions.swifthelper:SwiftInformationAction
|
||||
tripleo.templates.process = tripleo_common.actions.templates:ProcessTemplatesAction
|
||||
tripleo.templates.upload = tripleo_common.actions.templates:UploadTemplatesAction
|
||||
tripleo.templates.upload_plan_environment = tripleo_common.actions.templates:UploadPlanEnvironmentAction
|
||||
tripleo.validations.get_pubkey = tripleo_common.actions.validations:GetPubkeyAction
|
||||
|
|
|
@ -24,12 +24,12 @@ from mistral_lib import actions
|
|||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
from tripleo_common.actions import base
|
||||
from tripleo_common.actions import templates
|
||||
from tripleo_common import constants
|
||||
from tripleo_common import update
|
||||
from tripleo_common.utils import overcloudrc
|
||||
from tripleo_common.utils import plan as plan_utils
|
||||
from tripleo_common.utils import swift as swiftutils
|
||||
from tripleo_common.utils import template as template_utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -192,16 +192,9 @@ class DeployStackAction(base.TripleOAction):
|
|||
return actions.Result(error=err_msg)
|
||||
|
||||
# process all plan files and create or update a stack
|
||||
process_templates_action = templates.ProcessTemplatesAction(
|
||||
container=self.container
|
||||
processed_data = template_utils.process_templates(
|
||||
swift, heat, container=self.container
|
||||
)
|
||||
processed_data = process_templates_action.run(context)
|
||||
|
||||
# If we receive a 'Result' instance it is because the parent action
|
||||
# had an error.
|
||||
if isinstance(processed_data, actions.Result):
|
||||
return processed_data
|
||||
|
||||
stack_args = processed_data.copy()
|
||||
stack_args['timeout_mins'] = self.timeout_mins
|
||||
|
||||
|
|
|
@ -21,9 +21,9 @@ from mistral_lib import actions
|
|||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
from tripleo_common.actions import base
|
||||
from tripleo_common.actions import templates
|
||||
from tripleo_common import constants
|
||||
from tripleo_common.utils import plan as plan_utils
|
||||
from tripleo_common.utils import template as templates
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -38,6 +38,7 @@ class UpdateStackAction(base.TripleOAction):
|
|||
def run(self, context):
|
||||
# get the stack. Error if doesn't exist
|
||||
heat = self.get_orchestration_client(context)
|
||||
swift = self.get_object_client(context)
|
||||
try:
|
||||
stack = heat.stacks.get(self.container)
|
||||
except heat_exc.HTTPNotFound:
|
||||
|
@ -88,16 +89,9 @@ class UpdateStackAction(base.TripleOAction):
|
|||
return actions.Result(error=err_msg)
|
||||
|
||||
# process all plan files and create or update a stack
|
||||
process_templates_action = templates.ProcessTemplatesAction(
|
||||
container=self.container
|
||||
processed_data = templates.process_templates(
|
||||
swift, heat, container=self.container
|
||||
)
|
||||
processed_data = process_templates_action.run(context)
|
||||
|
||||
# If we receive a 'Result' instance it is because the parent action
|
||||
# had an error.
|
||||
if isinstance(processed_data, actions.Result):
|
||||
return processed_data
|
||||
|
||||
stack_args = processed_data.copy()
|
||||
|
||||
LOG.info("Performing Heat stack update")
|
||||
|
|
|
@ -23,13 +23,13 @@ from mistral_lib import actions
|
|||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
from tripleo_common.actions import base
|
||||
from tripleo_common.actions import templates
|
||||
from tripleo_common import constants
|
||||
from tripleo_common import exception
|
||||
from tripleo_common.utils import nodes
|
||||
from tripleo_common.utils import parameters as parameter_utils
|
||||
from tripleo_common.utils import passwords as password_utils
|
||||
from tripleo_common.utils import plan as plan_utils
|
||||
from tripleo_common.utils import template as template_utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -51,16 +51,9 @@ class GetParametersAction(base.TripleOAction):
|
|||
if cached is not None:
|
||||
return cached
|
||||
|
||||
process_templates_action = templates.ProcessTemplatesAction(
|
||||
container=self.container
|
||||
processed_data = template_utils.process_templates(
|
||||
swift, heat, container=self.container
|
||||
)
|
||||
processed_data = process_templates_action.run(context)
|
||||
|
||||
# If we receive a 'Result' instance it is because the parent action
|
||||
# had an error.
|
||||
if isinstance(processed_data, actions.Result):
|
||||
return processed_data
|
||||
|
||||
processed_data['show_nested'] = True
|
||||
|
||||
# respect previously user set param values
|
||||
|
@ -163,15 +156,9 @@ class UpdateParametersAction(base.TripleOAction):
|
|||
LOG.exception(err_msg)
|
||||
return actions.Result(error=err_msg)
|
||||
|
||||
process_templates_action = templates.ProcessTemplatesAction(
|
||||
container=self.container
|
||||
processed_data = template_utils.process_templates(
|
||||
swift, heat, container=self.container
|
||||
)
|
||||
processed_data = process_templates_action.run(context)
|
||||
|
||||
# If we receive a 'Result' instance it is because the parent action
|
||||
# had an error.
|
||||
if isinstance(processed_data, actions.Result):
|
||||
return processed_data
|
||||
|
||||
env = plan_utils.get_env(swift, self.container)
|
||||
if not self.validate:
|
||||
|
@ -208,7 +195,7 @@ class UpdateParametersAction(base.TripleOAction):
|
|||
# There has been an error validating we must reprocess the
|
||||
# templates with the saved working env
|
||||
plan_utils.put_env(swift, saved_env)
|
||||
process_templates_action._process_custom_roles(context)
|
||||
template_utils.process_custom_roles(swift, heat, self.container)
|
||||
|
||||
err_msg = ("Error validating environment for plan %s: %s" % (
|
||||
self.container, err))
|
||||
|
@ -668,15 +655,12 @@ class GetNetworkConfigAction(base.TripleOAction):
|
|||
self.role_name = role_name
|
||||
|
||||
def run(self, context):
|
||||
process_templates_action = templates.ProcessTemplatesAction(
|
||||
container=self.container
|
||||
)
|
||||
processed_data = process_templates_action.run(context)
|
||||
swift = self.get_object_client(context)
|
||||
heat = self.get_orchestration_client(context)
|
||||
|
||||
# If we receive a 'Result' instance it is because the parent action
|
||||
# had an error.
|
||||
if isinstance(processed_data, actions.Result):
|
||||
return processed_data
|
||||
processed_data = template_utils.process_templates(
|
||||
swift, heat, container=self.container
|
||||
)
|
||||
|
||||
# Default temporary value is used when no user input for any
|
||||
# interface routes for the role networks to find network config.
|
||||
|
|
|
@ -19,9 +19,9 @@ from mistral_lib import actions
|
|||
|
||||
from tripleo_common.actions import base
|
||||
from tripleo_common.actions import parameters as parameters_actions
|
||||
from tripleo_common.actions import templates
|
||||
from tripleo_common import constants
|
||||
from tripleo_common import update
|
||||
from tripleo_common.utils import template as template_utils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
@ -66,6 +66,8 @@ class ScaleDownAction(base.TripleOAction):
|
|||
def _update_stack(self, parameters={},
|
||||
timeout_mins=constants.STACK_TIMEOUT_DEFAULT,
|
||||
context=None):
|
||||
heat = self.get_orchestration_client(context)
|
||||
swift = self.get_object_client(context)
|
||||
# TODO(rbrady): migrate _update_stack to it's own action and update
|
||||
# the workflow for scale down
|
||||
|
||||
|
@ -76,13 +78,9 @@ class ScaleDownAction(base.TripleOAction):
|
|||
if isinstance(updated_plan, actions.Result):
|
||||
return updated_plan
|
||||
|
||||
process_templates_action = templates.ProcessTemplatesAction(
|
||||
container=self.container
|
||||
processed_data = template_utils.process_templates(
|
||||
swift, heat, container=self.container
|
||||
)
|
||||
processed_data = process_templates_action.run(context)
|
||||
if isinstance(processed_data, actions.Result):
|
||||
return processed_data
|
||||
|
||||
update.add_breakpoints_cleanup_into_env(processed_data['environment'])
|
||||
|
||||
fields = processed_data.copy()
|
||||
|
@ -95,8 +93,7 @@ class ScaleDownAction(base.TripleOAction):
|
|||
fields['clear_parameters'] = list(parameters.keys())
|
||||
|
||||
LOG.debug('stack update params: %s', fields)
|
||||
self.get_orchestration_client(context).stacks.update(self.container,
|
||||
**fields)
|
||||
heat.stacks.update(self.container, **fields)
|
||||
|
||||
def _get_removal_params_from_heat(self, resources_by_role, resources):
|
||||
stack_params = {}
|
||||
|
|
|
@ -12,59 +12,10 @@
|
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import jinja2
|
||||
import logging
|
||||
import os
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from heatclient import exc as heat_exc
|
||||
from mistral_lib import actions
|
||||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
from tripleo_common.actions import base
|
||||
from tripleo_common import constants
|
||||
from tripleo_common.utils import parameters
|
||||
from tripleo_common.utils import plan as plan_utils
|
||||
from tripleo_common.utils import swift as swiftutils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class J2SwiftLoader(jinja2.BaseLoader):
|
||||
"""Jinja2 loader to fetch included files from swift
|
||||
|
||||
This attempts to fetch a template include file from the given container.
|
||||
An optional search path or list of search paths can be provided. By default
|
||||
only the absolute path relative to the container root is searched.
|
||||
"""
|
||||
|
||||
def __init__(self, swift, container, searchpath=None):
|
||||
self.swift = swift
|
||||
self.container = container
|
||||
if searchpath is not None:
|
||||
if isinstance(searchpath, six.string_types):
|
||||
self.searchpath = [searchpath]
|
||||
else:
|
||||
self.searchpath = list(searchpath)
|
||||
else:
|
||||
self.searchpath = []
|
||||
# Always search the absolute path from the root of the swift container
|
||||
if '' not in self.searchpath:
|
||||
self.searchpath.append('')
|
||||
|
||||
def get_source(self, environment, template):
|
||||
pieces = jinja2.loaders.split_template_path(template)
|
||||
for searchpath in self.searchpath:
|
||||
template_path = os.path.join(searchpath, *pieces)
|
||||
try:
|
||||
source = swiftutils.get_object_string(self.swift,
|
||||
self.container,
|
||||
template_path)
|
||||
return source, None, False
|
||||
except swiftexceptions.ClientException:
|
||||
pass
|
||||
raise jinja2.exceptions.TemplateNotFound(template)
|
||||
|
||||
|
||||
class UploadTemplatesAction(base.UploadDirectoryAction):
|
||||
|
@ -87,318 +38,3 @@ class UploadPlanEnvironmentAction(base.TripleOAction):
|
|||
swift = self.get_object_client(context)
|
||||
# Push plan environment to the swift container
|
||||
plan_utils.put_env(swift, self.plan_environment)
|
||||
|
||||
|
||||
class ProcessTemplatesAction(base.TripleOAction):
|
||||
"""Process Templates and Environments
|
||||
|
||||
This method processes the templates and files in a given deployment
|
||||
plan into a format that can be passed to Heat.
|
||||
"""
|
||||
|
||||
def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
|
||||
super(ProcessTemplatesAction, self).__init__()
|
||||
self.container = container
|
||||
|
||||
def _j2_render_and_put(self,
|
||||
j2_template,
|
||||
j2_data,
|
||||
outfile_name=None,
|
||||
context=None):
|
||||
swift = self.get_object_client(context)
|
||||
yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
|
||||
|
||||
# Search for templates relative to the current template path first
|
||||
template_base = os.path.dirname(yaml_f)
|
||||
j2_loader = J2SwiftLoader(swift, self.container, template_base)
|
||||
|
||||
try:
|
||||
# Render the j2 template
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
j2_template)
|
||||
r_template = template.render(**j2_data)
|
||||
except jinja2.exceptions.TemplateError as ex:
|
||||
error_msg = ("Error rendering template %s : %s"
|
||||
% (yaml_f, six.text_type(ex)))
|
||||
LOG.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
try:
|
||||
# write the template back to the plan container
|
||||
LOG.info("Writing rendered template %s" % yaml_f)
|
||||
swiftutils.put_object_string(swift, self.container, yaml_f,
|
||||
r_template)
|
||||
except swiftexceptions.ClientException:
|
||||
error_msg = ("Error storing file %s in container %s"
|
||||
% (yaml_f, self.container))
|
||||
LOG.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
def _get_j2_excludes_file(self, context):
|
||||
swift = self.get_object_client(context)
|
||||
try:
|
||||
j2_excl_file = swiftutils.get_object_string(
|
||||
swift, self.container, constants.OVERCLOUD_J2_EXCLUDES)
|
||||
j2_excl_data = yaml.safe_load(j2_excl_file)
|
||||
if (j2_excl_data is None or j2_excl_data.get('name') is None):
|
||||
j2_excl_data = {"name": []}
|
||||
LOG.info("j2_excludes.yaml is either empty or there are "
|
||||
"no templates to exclude, defaulting the J2 "
|
||||
"excludes list to: %s" % j2_excl_data)
|
||||
except swiftexceptions.ClientException:
|
||||
j2_excl_data = {"name": []}
|
||||
LOG.info("No J2 exclude file found, defaulting "
|
||||
"the J2 excludes list to: %s" % j2_excl_data)
|
||||
return j2_excl_data
|
||||
|
||||
def _heat_resource_exists(self, heatclient, stack, nested_stack_name,
|
||||
resource_name, context):
|
||||
if stack is None:
|
||||
LOG.debug("Resource does not exist because stack does not exist")
|
||||
return False
|
||||
|
||||
try:
|
||||
nested_stack = heatclient.resources.get(stack.id,
|
||||
nested_stack_name)
|
||||
except heat_exc.HTTPNotFound:
|
||||
LOG.debug(
|
||||
"Resource does not exist because {} stack does "
|
||||
"not exist".format(nested_stack_name))
|
||||
return False
|
||||
|
||||
try:
|
||||
heatclient.resources.get(nested_stack.physical_resource_id,
|
||||
resource_name)
|
||||
except heat_exc.HTTPNotFound:
|
||||
LOG.debug("Resource does not exist: {}".format(resource_name))
|
||||
return False
|
||||
else:
|
||||
LOG.debug("Resource exists: {}".format(resource_name))
|
||||
return True
|
||||
|
||||
def _process_custom_roles(self, context):
|
||||
swift = self.get_object_client(context)
|
||||
|
||||
try:
|
||||
j2_role_file = swiftutils.get_object_string(
|
||||
swift, self.container, constants.OVERCLOUD_J2_ROLES_NAME)
|
||||
role_data = yaml.safe_load(j2_role_file)
|
||||
except swiftexceptions.ClientException:
|
||||
LOG.info("No %s file found, skipping jinja templating"
|
||||
% constants.OVERCLOUD_J2_ROLES_NAME)
|
||||
return
|
||||
|
||||
try:
|
||||
j2_network_file = swiftutils.get_object_string(
|
||||
swift, self.container, constants.OVERCLOUD_J2_NETWORKS_NAME)
|
||||
network_data = yaml.safe_load(j2_network_file)
|
||||
# Allow no networks defined in network_data
|
||||
if network_data is None:
|
||||
network_data = []
|
||||
except swiftexceptions.ClientException:
|
||||
# Until t-h-t contains network_data.yaml we tolerate a missing file
|
||||
LOG.warning("No %s file found, ignoring"
|
||||
% constants.OVERCLOUD_J2_ROLES_NAME)
|
||||
network_data = []
|
||||
|
||||
j2_excl_data = self._get_j2_excludes_file(context)
|
||||
|
||||
try:
|
||||
# Iterate over all files in the plan container
|
||||
# we j2 render any with the .j2.yaml suffix
|
||||
container_files = swift.get_container(self.container)
|
||||
except swiftexceptions.ClientException as ex:
|
||||
error_msg = ("Error listing contents of container %s : %s"
|
||||
% (self.container, six.text_type(ex)))
|
||||
LOG.error(error_msg)
|
||||
raise Exception(error_msg)
|
||||
|
||||
role_names = [r.get('name') for r in role_data]
|
||||
r_map = {}
|
||||
for r in role_data:
|
||||
r_map[r.get('name')] = r
|
||||
excl_templates = j2_excl_data.get('name')
|
||||
|
||||
heatclient = self.get_orchestration_client(context)
|
||||
stack = None
|
||||
try:
|
||||
stack = heatclient.stacks.get(self.container,
|
||||
resolve_outputs=False)
|
||||
except heat_exc.HTTPNotFound:
|
||||
LOG.debug("Stack does not exist")
|
||||
|
||||
n_map = {}
|
||||
for n in network_data:
|
||||
if n.get('enabled') is not False:
|
||||
n_map[n.get('name')] = n
|
||||
if not n.get('name_lower'):
|
||||
n_map[n.get('name')]['name_lower'] = n.get('name').lower()
|
||||
if n.get('name') == constants.API_NETWORK and 'compat_name' \
|
||||
not in n.keys():
|
||||
# Check to see if legacy named API network exists
|
||||
# and if so we need to set compat_name
|
||||
api_net = "{}Network".format(constants.LEGACY_API_NETWORK)
|
||||
if self._heat_resource_exists(heatclient, stack, 'Networks',
|
||||
api_net, context):
|
||||
n['compat_name'] = 'Internal'
|
||||
LOG.info("Upgrade compatibility enabled for legacy "
|
||||
"network resource Internal.")
|
||||
else:
|
||||
LOG.info("skipping %s network: network is disabled." %
|
||||
n.get('name'))
|
||||
|
||||
plan_utils.cache_delete(swift, self.container,
|
||||
"tripleo.parameters.get")
|
||||
|
||||
for f in [f.get('name') for f in container_files[1]]:
|
||||
# We do three templating passes here:
|
||||
# 1. *.role.j2.yaml - we template just the role name
|
||||
# and create multiple files (one per role)
|
||||
# 2 *.network.j2.yaml - we template the network name and
|
||||
# data and create multiple files for networks and
|
||||
# network ports (one per network)
|
||||
# 3. *.j2.yaml - we template with all roles_data,
|
||||
# and create one file common to all roles
|
||||
if f.endswith('.role.j2.yaml'):
|
||||
LOG.info("jinja2 rendering role template %s" % f)
|
||||
j2_template = swiftutils.get_object_string(swift,
|
||||
self.container,
|
||||
f)
|
||||
LOG.info("jinja2 rendering roles %s" % ","
|
||||
.join(role_names))
|
||||
for role in role_names:
|
||||
LOG.info("jinja2 rendering role %s" % role)
|
||||
out_f = "-".join(
|
||||
[role.lower(),
|
||||
os.path.basename(f).replace('.role.j2.yaml',
|
||||
'.yaml')])
|
||||
out_f_path = os.path.join(os.path.dirname(f), out_f)
|
||||
if ('network/config' in os.path.dirname(f) and
|
||||
r_map[role].get('deprecated_nic_config_name')):
|
||||
d_name = r_map[role].get('deprecated_nic_config_name')
|
||||
out_f_path = os.path.join(os.path.dirname(f), d_name)
|
||||
elif ('network/config' in os.path.dirname(f)):
|
||||
d_name = "%s.yaml" % role.lower()
|
||||
out_f_path = os.path.join(os.path.dirname(f), d_name)
|
||||
if not (out_f_path in excl_templates):
|
||||
if '{{role.name}}' in j2_template:
|
||||
j2_data = {'role': r_map[role],
|
||||
'networks': network_data}
|
||||
self._j2_render_and_put(j2_template,
|
||||
j2_data,
|
||||
out_f_path,
|
||||
context=context)
|
||||
else:
|
||||
# Backwards compatibility with templates
|
||||
# that specify {{role}} vs {{role.name}}
|
||||
j2_data = {'role': role, 'networks': network_data}
|
||||
LOG.debug("role legacy path for role %s" % role)
|
||||
self._j2_render_and_put(j2_template,
|
||||
j2_data,
|
||||
out_f_path,
|
||||
context=context)
|
||||
else:
|
||||
LOG.info("Skipping rendering of %s, defined in %s" %
|
||||
(out_f_path, j2_excl_data))
|
||||
|
||||
elif (f.endswith('.network.j2.yaml')):
|
||||
LOG.info("jinja2 rendering network template %s" % f)
|
||||
j2_template = swiftutils.get_object_string(swift,
|
||||
self.container,
|
||||
f)
|
||||
LOG.info("jinja2 rendering networks %s" % ",".join(n_map))
|
||||
for network in n_map:
|
||||
j2_data = {'network': n_map[network]}
|
||||
# Output file names in "<name>.yaml" format
|
||||
out_f = os.path.basename(f).replace('.network.j2.yaml',
|
||||
'.yaml')
|
||||
if os.path.dirname(f).endswith('ports'):
|
||||
out_f = out_f.replace('port',
|
||||
n_map[network]['name_lower'])
|
||||
else:
|
||||
out_f = out_f.replace('network',
|
||||
n_map[network]['name_lower'])
|
||||
out_f_path = os.path.join(os.path.dirname(f), out_f)
|
||||
if not (out_f_path in excl_templates):
|
||||
self._j2_render_and_put(j2_template,
|
||||
j2_data,
|
||||
out_f_path,
|
||||
context=context)
|
||||
else:
|
||||
LOG.info("Skipping rendering of %s, defined in %s" %
|
||||
(out_f_path, j2_excl_data))
|
||||
|
||||
elif f.endswith('.j2.yaml'):
|
||||
LOG.info("jinja2 rendering %s" % f)
|
||||
j2_template = swiftutils.get_object_string(swift,
|
||||
self.container,
|
||||
f)
|
||||
j2_data = {'roles': role_data, 'networks': network_data}
|
||||
out_f = f.replace('.j2.yaml', '.yaml')
|
||||
self._j2_render_and_put(j2_template,
|
||||
j2_data,
|
||||
out_f,
|
||||
context=context)
|
||||
|
||||
def run(self, context):
|
||||
error_text = None
|
||||
self.context = context
|
||||
swift = self.get_object_client(context)
|
||||
|
||||
try:
|
||||
plan_env = plan_utils.get_env(swift, self.container)
|
||||
except swiftexceptions.ClientException as err:
|
||||
err_msg = ("Error retrieving environment for plan %s: %s" % (
|
||||
self.container, err))
|
||||
LOG.exception(err_msg)
|
||||
return actions.Result(error=error_text)
|
||||
|
||||
try:
|
||||
# if the jinja overcloud template exists, process it and write it
|
||||
# back to the swift container before continuing processing. The
|
||||
# method called below should handle the case where the files are
|
||||
# not found in swift, but if they are found and an exception
|
||||
# occurs during processing, that exception will cause the
|
||||
# ProcessTemplatesAction to return an error result.
|
||||
self._process_custom_roles(context)
|
||||
except Exception as err:
|
||||
LOG.exception("Error occurred while processing custom roles.")
|
||||
return actions.Result(error=six.text_type(err))
|
||||
|
||||
template_name = plan_env.get('template', "")
|
||||
|
||||
template_object = os.path.join(swift.url, self.container,
|
||||
template_name)
|
||||
LOG.debug('Template: %s' % template_name)
|
||||
try:
|
||||
template_files, template = plan_utils.get_template_contents(
|
||||
swift, template_object)
|
||||
except Exception as err:
|
||||
error_text = six.text_type(err)
|
||||
LOG.exception("Error occurred while fetching %s" % template_object)
|
||||
|
||||
temp_env_paths = []
|
||||
try:
|
||||
env_paths, temp_env_paths = plan_utils.build_env_paths(
|
||||
swift, self.container, plan_env)
|
||||
env_files, env = plan_utils.process_environments_and_files(
|
||||
swift, env_paths)
|
||||
parameters.convert_docker_params(env)
|
||||
except Exception as err:
|
||||
error_text = six.text_type(err)
|
||||
LOG.exception("Error occurred while processing plan files.")
|
||||
finally:
|
||||
# cleanup any local temp files
|
||||
for f in temp_env_paths:
|
||||
os.remove(f)
|
||||
|
||||
if error_text:
|
||||
return actions.Result(error=error_text)
|
||||
|
||||
files = dict(list(template_files.items()) + list(env_files.items()))
|
||||
|
||||
return {
|
||||
'stack_name': self.container,
|
||||
'template': template,
|
||||
'environment': env,
|
||||
'files': files
|
||||
}
|
||||
|
|
|
@ -25,7 +25,7 @@ class UpdateStackActionTest(base.TestCase):
|
|||
self.timeout = 1
|
||||
self.container = 'container'
|
||||
|
||||
@mock.patch('tripleo_common.actions.templates.ProcessTemplatesAction.run')
|
||||
@mock.patch('tripleo_common.utils.template.process_templates')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.'
|
||||
'get_orchestration_client')
|
||||
|
@ -40,7 +40,7 @@ class UpdateStackActionTest(base.TestCase):
|
|||
mock_template_contents,
|
||||
mock_get_orchestration_client,
|
||||
mock_get_object_client,
|
||||
mock_templates_run):
|
||||
mock_process_templates):
|
||||
mock_ctx = mock.MagicMock()
|
||||
|
||||
heat = mock.MagicMock()
|
||||
|
|
|
@ -12,12 +12,7 @@
|
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import jinja2
|
||||
import mock
|
||||
import yaml
|
||||
|
||||
from heatclient import exc as heat_exc
|
||||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
from tripleo_common.actions import templates
|
||||
from tripleo_common import constants
|
||||
|
@ -119,418 +114,3 @@ class UploadTemplatesActionTest(base.TestCase):
|
|||
constants.DEFAULT_TEMPLATES_PATH, 'test')
|
||||
mock_extract_tar.assert_called_once_with(
|
||||
mock_get_swift.return_value, 'test', 'tar-container')
|
||||
|
||||
|
||||
class J2SwiftLoaderTest(base.TestCase):
|
||||
@staticmethod
|
||||
def _setup_swift():
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == 'bar/foo.yaml':
|
||||
return ['', 'I am foo']
|
||||
else:
|
||||
raise swiftexceptions.ClientException('not found')
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
return swift
|
||||
|
||||
def test_include_absolute_path(self):
|
||||
j2_loader = templates.J2SwiftLoader(self._setup_swift(), None)
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include 'bar/foo.yaml' %}
|
||||
''')
|
||||
self.assertEqual(
|
||||
template.render(),
|
||||
'''
|
||||
Included this:
|
||||
I am foo
|
||||
''')
|
||||
|
||||
def test_include_search_path(self):
|
||||
j2_loader = templates.J2SwiftLoader(self._setup_swift(), None, 'bar')
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include 'foo.yaml' %}
|
||||
''')
|
||||
self.assertEqual(
|
||||
template.render(),
|
||||
'''
|
||||
Included this:
|
||||
I am foo
|
||||
''')
|
||||
|
||||
def test_include_not_found(self):
|
||||
j2_loader = templates.J2SwiftLoader(self._setup_swift(), None)
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include 'bar.yaml' %}
|
||||
''')
|
||||
self.assertRaises(
|
||||
jinja2.exceptions.TemplateNotFound,
|
||||
template.render)
|
||||
|
||||
def test_include_invalid_path(self):
|
||||
j2_loader = templates.J2SwiftLoader(self._setup_swift(), 'bar')
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include '../foo.yaml' %}
|
||||
''')
|
||||
self.assertRaises(
|
||||
jinja2.exceptions.TemplateNotFound,
|
||||
template.render)
|
||||
|
||||
|
||||
class ProcessTemplatesActionTest(base.TestCase):
|
||||
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_multiple_environments_and_files')
|
||||
@mock.patch('heatclient.common.template_utils.get_template_contents')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.'
|
||||
'get_orchestration_client')
|
||||
def test_run(self, mock_get_heat_client, mock_get_object_client,
|
||||
mock_get_template_contents,
|
||||
mock_process_multiple_environments_and_files):
|
||||
|
||||
mock_ctx = mock.MagicMock()
|
||||
swift = mock.MagicMock(url="http://test.com")
|
||||
mock_env = yaml.safe_dump({
|
||||
'temp_environment': 'temp_environment',
|
||||
'template': 'template',
|
||||
'environments': [{u'path': u'environments/test.yaml'}]
|
||||
}, default_flow_style=False)
|
||||
swift.get_object.side_effect = (
|
||||
({}, mock_env),
|
||||
swiftexceptions.ClientException('atest2')
|
||||
)
|
||||
mock_get_object_client.return_value = swift
|
||||
|
||||
mock_get_template_contents.return_value = ({}, {
|
||||
'heat_template_version': '2016-04-30'
|
||||
})
|
||||
mock_process_multiple_environments_and_files.return_value = ({}, {})
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
result = action.run(mock_ctx)
|
||||
|
||||
# Verify the values we get out
|
||||
self.assertEqual(result, {
|
||||
'environment': {},
|
||||
'files': {},
|
||||
'stack_name': constants.DEFAULT_CONTAINER_NAME,
|
||||
'template': {
|
||||
'heat_template_version': '2016-04-30'
|
||||
}
|
||||
})
|
||||
|
||||
def _custom_roles_mock_objclient(self, snippet_name, snippet_content,
|
||||
role_data=None):
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_NAME:
|
||||
return ['', JINJA_SNIPPET]
|
||||
if args[1] == snippet_name:
|
||||
return ['', snippet_content]
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES]
|
||||
elif args[1] == constants.OVERCLOUD_J2_ROLES_NAME:
|
||||
return ['', role_data or ROLE_DATA_YAML]
|
||||
elif args[1] == constants.OVERCLOUD_J2_NETWORKS_NAME:
|
||||
return ['', NETWORK_DATA_YAML]
|
||||
|
||||
def return_container_files(*args):
|
||||
return ('headers', [
|
||||
{'name': constants.OVERCLOUD_J2_NAME},
|
||||
{'name': snippet_name},
|
||||
{'name': constants.OVERCLOUD_J2_ROLES_NAME},
|
||||
{'name': constants.OVERCLOUD_J2_NETWORKS_NAME}])
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
swift.get_container = mock.MagicMock(
|
||||
side_effect=return_container_files)
|
||||
return swift
|
||||
|
||||
@mock.patch('tripleo_common.actions.templates.ProcessTemplatesAction'
|
||||
'._heat_resource_exists')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.'
|
||||
'get_orchestration_client')
|
||||
def test_process_custom_roles(self, get_heat_client_mock,
|
||||
get_obj_client_mock, resource_exists_mock):
|
||||
resource_exists_mock.return_value = False
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'foo.j2.yaml', JINJA_SNIPPET)
|
||||
get_obj_client_mock.return_value = swift
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
mock_ctx = mock.MagicMock()
|
||||
action._process_custom_roles(mock_ctx)
|
||||
|
||||
get_object_mock_calls = [
|
||||
mock.call('overcloud', constants.OVERCLOUD_J2_NAME),
|
||||
mock.call('overcloud', constants.OVERCLOUD_J2_ROLES_NAME),
|
||||
mock.call('overcloud', 'foo.j2.yaml'),
|
||||
]
|
||||
swift.get_object.assert_has_calls(
|
||||
get_object_mock_calls, any_order=True)
|
||||
|
||||
put_object_mock_calls = [
|
||||
mock.call(constants.DEFAULT_CONTAINER_NAME,
|
||||
constants.OVERCLOUD_YAML_NAME,
|
||||
EXPECTED_JINJA_RESULT),
|
||||
mock.call(constants.DEFAULT_CONTAINER_NAME,
|
||||
'foo.yaml',
|
||||
EXPECTED_JINJA_RESULT),
|
||||
]
|
||||
swift.put_object.assert_has_calls(
|
||||
put_object_mock_calls, any_order=True)
|
||||
|
||||
@mock.patch('tripleo_common.actions.templates.ProcessTemplatesAction'
|
||||
'._heat_resource_exists')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction'
|
||||
'.get_orchestration_client')
|
||||
def test_custom_roles_networks(self, get_heat_client_mock,
|
||||
get_obj_client_mock, resource_exists_mock):
|
||||
resource_exists_mock.return_value = False
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'role-networks.role.j2.yaml', JINJA_SNIPPET_ROLE_NETWORKS,
|
||||
ROLE_DATA_ENABLE_NETWORKS)
|
||||
get_obj_client_mock.return_value = swift
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
mock_ctx = mock.MagicMock()
|
||||
action._process_custom_roles(mock_ctx)
|
||||
|
||||
expected = EXPECTED_JINJA_RESULT.replace(
|
||||
'CustomRole', 'RoleWithNetworks')
|
||||
put_object_mock_call = mock.call(
|
||||
constants.DEFAULT_CONTAINER_NAME,
|
||||
'overcloud.yaml',
|
||||
expected)
|
||||
self.assertEqual(swift.put_object.call_args_list[0],
|
||||
put_object_mock_call)
|
||||
|
||||
put_object_mock_call = mock.call(
|
||||
constants.DEFAULT_CONTAINER_NAME,
|
||||
"rolewithnetworks-role-networks.yaml",
|
||||
EXPECTED_JINJA_RESULT_ROLE_NETWORKS)
|
||||
self.assertEqual(put_object_mock_call,
|
||||
swift.put_object.call_args_list[1])
|
||||
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
def test_j2_render_and_put(self, get_obj_client_mock):
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock()
|
||||
swift.get_container = mock.MagicMock()
|
||||
get_obj_client_mock.return_value = swift
|
||||
mock_ctx = mock.MagicMock()
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
action._j2_render_and_put(JINJA_SNIPPET_CONFIG,
|
||||
{'role': 'CustomRole'},
|
||||
'customrole-config.yaml', context=mock_ctx)
|
||||
|
||||
action_result = swift.put_object._mock_mock_calls[0]
|
||||
|
||||
self.assertTrue("CustomRole" in str(action_result))
|
||||
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
def test_j2_render_and_put_include(self, get_obj_client_mock):
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == 'foo.yaml':
|
||||
return ['', JINJA_SNIPPET_CONFIG]
|
||||
|
||||
def return_container_files(*args):
|
||||
return ('headers', [{'name': 'foo.yaml'}])
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
swift.get_container = mock.MagicMock(
|
||||
side_effect=return_container_files)
|
||||
get_obj_client_mock.return_value = swift
|
||||
mock_ctx = mock.MagicMock()
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
action._j2_render_and_put(r"{% include 'foo.yaml' %}",
|
||||
{'role': 'CustomRole'},
|
||||
'customrole-config.yaml',
|
||||
context=mock_ctx)
|
||||
|
||||
action_result = swift.put_object._mock_mock_calls[0]
|
||||
|
||||
self.assertTrue("CustomRole" in str(action_result))
|
||||
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
def test_j2_render_and_put_include_relative(
|
||||
self,
|
||||
get_obj_client_mock):
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == 'bar/foo.yaml':
|
||||
return ['', JINJA_SNIPPET_CONFIG]
|
||||
|
||||
def return_container_files(*args):
|
||||
return ('headers', [{'name': 'bar/foo.yaml'}])
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
swift.get_container = mock.MagicMock(
|
||||
side_effect=return_container_files)
|
||||
get_obj_client_mock.return_value = swift
|
||||
mock_ctx = mock.MagicMock()
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
action._j2_render_and_put(r"{% include 'foo.yaml' %}",
|
||||
{'role': 'CustomRole'},
|
||||
'bar/customrole-config.yaml',
|
||||
context=mock_ctx)
|
||||
|
||||
action_result = swift.put_object._mock_mock_calls[0]
|
||||
|
||||
self.assertTrue("CustomRole" in str(action_result))
|
||||
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
def test_get_j2_excludes_file(self, get_obj_client_mock):
|
||||
|
||||
mock_ctx = mock.MagicMock()
|
||||
swift = mock.MagicMock()
|
||||
get_obj_client_mock.return_value = swift
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES]
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
# Test - J2 exclude file with valid templates
|
||||
action = templates.ProcessTemplatesAction()
|
||||
self.assertTrue({'name': ['puppet/controller-role.yaml']} ==
|
||||
action._get_j2_excludes_file(mock_ctx))
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES_EMPTY_LIST]
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
# Test - J2 exclude file with no template to exlude
|
||||
action = templates.ProcessTemplatesAction()
|
||||
self.assertTrue({'name': []} == action._get_j2_excludes_file(mock_ctx))
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES_EMPTY_FILE]
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
# Test - J2 exclude file empty
|
||||
action = templates.ProcessTemplatesAction()
|
||||
self.assertTrue({'name': []} == action._get_j2_excludes_file(mock_ctx))
|
||||
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.'
|
||||
'get_orchestration_client')
|
||||
def test_heat_resource_exists(self, client_mock):
|
||||
mock_ctx = mock.MagicMock()
|
||||
heat_client = mock.MagicMock()
|
||||
stack = mock.MagicMock(stack_name='overcloud')
|
||||
heat_client.resources.get.return_value = mock.MagicMock(
|
||||
links=[{'rel': 'stack',
|
||||
'href': 'http://192.0.2.1:8004/v1/'
|
||||
'a959ac7d6a4a475daf2428df315c41ef/'
|
||||
'stacks/overcloud/123'}],
|
||||
logical_resource_id='logical_id',
|
||||
physical_resource_id='resource_id',
|
||||
resource_type='OS::Heat::ResourceGroup',
|
||||
resource_name='InternalApiNetwork'
|
||||
)
|
||||
client_mock.return_value = heat_client
|
||||
action = templates.ProcessTemplatesAction()
|
||||
self.assertTrue(
|
||||
action._heat_resource_exists(
|
||||
heat_client, stack, 'Networks', 'InternalNetwork', mock_ctx))
|
||||
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.'
|
||||
'get_orchestration_client')
|
||||
def test_no_heat_resource_exists(self, client_mock):
|
||||
mock_ctx = mock.MagicMock()
|
||||
heat_client = mock.MagicMock()
|
||||
stack = mock.MagicMock(stack_name='overcloud')
|
||||
|
||||
def return_not_found(*args):
|
||||
raise heat_exc.HTTPNotFound()
|
||||
|
||||
heat_client.resources.get.side_effect = return_not_found
|
||||
client_mock.return_value = heat_client
|
||||
action = templates.ProcessTemplatesAction()
|
||||
self.assertFalse(
|
||||
action._heat_resource_exists(
|
||||
heat_client, stack, 'Networks', 'InternalNetwork', mock_ctx))
|
||||
|
||||
@mock.patch('tripleo_common.actions.templates.ProcessTemplatesAction'
|
||||
'._heat_resource_exists')
|
||||
@mock.patch('tripleo_common.actions.templates.ProcessTemplatesAction'
|
||||
'._j2_render_and_put')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.'
|
||||
'get_orchestration_client')
|
||||
def test_legacy_api_network_exists(self, get_heat_client_mock,
|
||||
get_obj_client_mock, j2_mock,
|
||||
resource_exists_mock):
|
||||
resource_exists_mock.return_value = True
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'role-networks.role.j2.yaml', JINJA_SNIPPET_ROLE_NETWORKS,
|
||||
ROLE_DATA_ENABLE_NETWORKS)
|
||||
get_obj_client_mock.return_value = swift
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
mock_ctx = mock.MagicMock()
|
||||
action._process_custom_roles(mock_ctx)
|
||||
expected_j2_template = get_obj_client_mock.get_object(
|
||||
action.container, 'foo.j2.yaml')[1]
|
||||
expected_j2_data = {'roles': [{'name': 'CustomRole'}],
|
||||
'networks': [{'name': 'InternalApi',
|
||||
'compat_name': 'Internal'}]
|
||||
}
|
||||
assert j2_mock.called_with(expected_j2_template, expected_j2_data,
|
||||
'foo.yaml', mock_ctx)
|
||||
|
||||
@mock.patch('tripleo_common.actions.templates.ProcessTemplatesAction'
|
||||
'._heat_resource_exists')
|
||||
@mock.patch('tripleo_common.actions.templates.ProcessTemplatesAction'
|
||||
'._j2_render_and_put')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.get_object_client')
|
||||
@mock.patch('tripleo_common.actions.base.TripleOAction.'
|
||||
'get_orchestration_client')
|
||||
def test_no_legacy_api_network_exists(self, get_heat_client_mock,
|
||||
get_obj_client_mock, j2_mock,
|
||||
resource_exists_mock):
|
||||
resource_exists_mock.return_value = False
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'role-networks.role.j2.yaml', JINJA_SNIPPET_ROLE_NETWORKS,
|
||||
ROLE_DATA_ENABLE_NETWORKS)
|
||||
get_obj_client_mock.return_value = swift
|
||||
|
||||
# Test
|
||||
action = templates.ProcessTemplatesAction()
|
||||
mock_ctx = mock.MagicMock()
|
||||
action._process_custom_roles(mock_ctx)
|
||||
expected_j2_template = get_obj_client_mock.get_object(
|
||||
action.container, 'foo.j2.yaml')[1]
|
||||
expected_j2_data = {'roles': [{'name': 'CustomRole'}],
|
||||
'networks': [{'name': 'InternalApi'}]
|
||||
}
|
||||
assert j2_mock.called_with(expected_j2_template, expected_j2_data,
|
||||
'foo.yaml', mock_ctx)
|
||||
|
|
|
@ -0,0 +1,460 @@
|
|||
# Copyright 2016 Red Hat, Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import jinja2
|
||||
import mock
|
||||
import yaml
|
||||
|
||||
from heatclient import exc as heat_exc
|
||||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
from tripleo_common import constants
|
||||
from tripleo_common.tests import base
|
||||
from tripleo_common.utils import template as template_utils
|
||||
|
||||
JINJA_SNIPPET = r"""
|
||||
# Jinja loop for Role in role_data.yaml
|
||||
{% for role in roles %}
|
||||
# Resources generated for {{role.name}} Role
|
||||
{{role.name}}ServiceChain:
|
||||
type: OS::TripleO::Services
|
||||
properties:
|
||||
Services:
|
||||
get_param: {{role.name}}Services
|
||||
ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
|
||||
EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
|
||||
DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
|
||||
{% endfor %}"""
|
||||
|
||||
ROLE_DATA_YAML = r"""
|
||||
-
|
||||
name: CustomRole
|
||||
"""
|
||||
|
||||
NETWORK_DATA_YAML = r"""
|
||||
-
|
||||
name: InternalApi
|
||||
"""
|
||||
|
||||
EXPECTED_JINJA_RESULT = r"""
|
||||
# Jinja loop for Role in role_data.yaml
|
||||
|
||||
# Resources generated for CustomRole Role
|
||||
CustomRoleServiceChain:
|
||||
type: OS::TripleO::Services
|
||||
properties:
|
||||
Services:
|
||||
get_param: CustomRoleServices
|
||||
ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map]}
|
||||
EndpointMap: {get_attr: [EndpointMap, endpoint_map]}
|
||||
DefaultPasswords: {get_attr: [DefaultPasswords, passwords]}
|
||||
"""
|
||||
|
||||
JINJA_SNIPPET_CONFIG = r"""
|
||||
outputs:
|
||||
OS::stack_id:
|
||||
description: The software config which runs puppet on the {{role}} role
|
||||
value: {get_resource: {{role}}PuppetConfigImpl}"""
|
||||
|
||||
J2_EXCLUDES = r"""
|
||||
name:
|
||||
- puppet/controller-role.yaml
|
||||
"""
|
||||
|
||||
J2_EXCLUDES_EMPTY_LIST = r"""
|
||||
name:
|
||||
"""
|
||||
|
||||
J2_EXCLUDES_EMPTY_FILE = r"""
|
||||
"""
|
||||
|
||||
ROLE_DATA_ENABLE_NETWORKS = r"""
|
||||
- name: RoleWithNetworks
|
||||
networks:
|
||||
- InternalApi
|
||||
"""
|
||||
|
||||
JINJA_SNIPPET_ROLE_NETWORKS = r"""
|
||||
{%- for network in networks %}
|
||||
{%- if network.name in role.networks%}
|
||||
{{network.name}}Port:
|
||||
type: {{role.name}}::{{network.name}}::Port
|
||||
{%- endif %}
|
||||
{% endfor %}
|
||||
"""
|
||||
|
||||
EXPECTED_JINJA_RESULT_ROLE_NETWORKS = r"""
|
||||
InternalApiPort:
|
||||
type: RoleWithNetworks::InternalApi::Port
|
||||
"""
|
||||
|
||||
|
||||
class J2SwiftLoaderTest(base.TestCase):
|
||||
@staticmethod
|
||||
def _setup_swift():
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == 'bar/foo.yaml':
|
||||
return ['', 'I am foo']
|
||||
else:
|
||||
raise swiftexceptions.ClientException('not found')
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
return swift
|
||||
|
||||
def test_include_absolute_path(self):
|
||||
j2_loader = template_utils.J2SwiftLoader(self._setup_swift(), None)
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include 'bar/foo.yaml' %}
|
||||
''')
|
||||
self.assertEqual(
|
||||
template.render(),
|
||||
'''
|
||||
Included this:
|
||||
I am foo
|
||||
''')
|
||||
|
||||
def test_include_search_path(self):
|
||||
j2_loader = template_utils.J2SwiftLoader(self._setup_swift(),
|
||||
None, 'bar')
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include 'foo.yaml' %}
|
||||
''')
|
||||
self.assertEqual(
|
||||
template.render(),
|
||||
'''
|
||||
Included this:
|
||||
I am foo
|
||||
''')
|
||||
|
||||
def test_include_not_found(self):
|
||||
j2_loader = template_utils.J2SwiftLoader(self._setup_swift(), None)
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include 'bar.yaml' %}
|
||||
''')
|
||||
self.assertRaises(
|
||||
jinja2.exceptions.TemplateNotFound,
|
||||
template.render)
|
||||
|
||||
def test_include_invalid_path(self):
|
||||
j2_loader = template_utils.J2SwiftLoader(self._setup_swift(), 'bar')
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
r'''
|
||||
Included this:
|
||||
{% include '../foo.yaml' %}
|
||||
''')
|
||||
self.assertRaises(
|
||||
jinja2.exceptions.TemplateNotFound,
|
||||
template.render)
|
||||
|
||||
|
||||
class ProcessTemplatesTest(base.TestCase):
|
||||
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_multiple_environments_and_files')
|
||||
@mock.patch('heatclient.common.template_utils.get_template_contents')
|
||||
def test_process_template(self, mock_get_template_contents,
|
||||
mock_process_multiple_environments_and_files):
|
||||
|
||||
swift = mock.MagicMock(url="http://test.com")
|
||||
heat = mock.MagicMock()
|
||||
mock_env = yaml.safe_dump({
|
||||
'temp_environment': 'temp_environment',
|
||||
'template': 'template',
|
||||
'environments': [{u'path': u'environments/test.yaml'}]
|
||||
}, default_flow_style=False)
|
||||
swift.get_object.side_effect = (
|
||||
({}, mock_env),
|
||||
swiftexceptions.ClientException('atest2')
|
||||
)
|
||||
mock_get_template_contents.return_value = ({}, {
|
||||
'heat_template_version': '2016-04-30'
|
||||
})
|
||||
mock_process_multiple_environments_and_files.return_value = ({}, {})
|
||||
|
||||
# Test
|
||||
processed_data = template_utils.process_templates(
|
||||
swift, heat, constants.DEFAULT_CONTAINER_NAME)
|
||||
|
||||
# Verify the values we get out
|
||||
self.assertEqual(processed_data, {
|
||||
'environment': {},
|
||||
'files': {},
|
||||
'stack_name': constants.DEFAULT_CONTAINER_NAME,
|
||||
'template': {
|
||||
'heat_template_version': '2016-04-30'
|
||||
}
|
||||
})
|
||||
|
||||
def _custom_roles_mock_objclient(self, snippet_name, snippet_content,
|
||||
role_data=None):
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_NAME:
|
||||
return ['', JINJA_SNIPPET]
|
||||
if args[1] == snippet_name:
|
||||
return ['', snippet_content]
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES]
|
||||
elif args[1] == constants.OVERCLOUD_J2_ROLES_NAME:
|
||||
return ['', role_data or ROLE_DATA_YAML]
|
||||
elif args[1] == constants.OVERCLOUD_J2_NETWORKS_NAME:
|
||||
return ['', NETWORK_DATA_YAML]
|
||||
|
||||
def return_container_files(*args):
|
||||
return ('headers', [
|
||||
{'name': constants.OVERCLOUD_J2_NAME},
|
||||
{'name': snippet_name},
|
||||
{'name': constants.OVERCLOUD_J2_ROLES_NAME},
|
||||
{'name': constants.OVERCLOUD_J2_NETWORKS_NAME}])
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
swift.get_container = mock.MagicMock(
|
||||
side_effect=return_container_files)
|
||||
return swift
|
||||
|
||||
@mock.patch('tripleo_common.utils.template.heat_resource_exists')
|
||||
def test_process_custom_roles(self, resource_exists_mock):
|
||||
resource_exists_mock.return_value = False
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'foo.j2.yaml', JINJA_SNIPPET)
|
||||
heat = mock.MagicMock()
|
||||
|
||||
# Test
|
||||
template_utils.process_custom_roles(
|
||||
swift, heat, constants.DEFAULT_CONTAINER_NAME)
|
||||
|
||||
get_object_mock_calls = [
|
||||
mock.call('overcloud', constants.OVERCLOUD_J2_NAME),
|
||||
mock.call('overcloud', constants.OVERCLOUD_J2_ROLES_NAME),
|
||||
mock.call('overcloud', 'foo.j2.yaml'),
|
||||
]
|
||||
swift.get_object.assert_has_calls(
|
||||
get_object_mock_calls, any_order=True)
|
||||
|
||||
put_object_mock_calls = [
|
||||
mock.call(constants.DEFAULT_CONTAINER_NAME,
|
||||
constants.OVERCLOUD_YAML_NAME,
|
||||
EXPECTED_JINJA_RESULT),
|
||||
mock.call(constants.DEFAULT_CONTAINER_NAME,
|
||||
'foo.yaml',
|
||||
EXPECTED_JINJA_RESULT),
|
||||
]
|
||||
swift.put_object.assert_has_calls(
|
||||
put_object_mock_calls, any_order=True)
|
||||
|
||||
@mock.patch('tripleo_common.utils.template.heat_resource_exists')
|
||||
def test_custom_roles_networks(self, resource_exists_mock):
|
||||
heat = mock.MagicMock()
|
||||
resource_exists_mock.return_value = False
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'role-networks.role.j2.yaml', JINJA_SNIPPET_ROLE_NETWORKS,
|
||||
ROLE_DATA_ENABLE_NETWORKS)
|
||||
|
||||
# Test
|
||||
template_utils.process_custom_roles(
|
||||
swift, heat, constants.DEFAULT_CONTAINER_NAME)
|
||||
|
||||
expected = EXPECTED_JINJA_RESULT.replace(
|
||||
'CustomRole', 'RoleWithNetworks')
|
||||
put_object_mock_call = mock.call(
|
||||
constants.DEFAULT_CONTAINER_NAME,
|
||||
'overcloud.yaml',
|
||||
expected)
|
||||
self.assertEqual(swift.put_object.call_args_list[0],
|
||||
put_object_mock_call)
|
||||
|
||||
put_object_mock_call = mock.call(
|
||||
constants.DEFAULT_CONTAINER_NAME,
|
||||
"rolewithnetworks-role-networks.yaml",
|
||||
EXPECTED_JINJA_RESULT_ROLE_NETWORKS)
|
||||
self.assertEqual(put_object_mock_call,
|
||||
swift.put_object.call_args_list[1])
|
||||
|
||||
def test_j2_render_and_put(self):
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock()
|
||||
swift.get_container = mock.MagicMock()
|
||||
|
||||
# Test
|
||||
template_utils.j2_render_and_put(
|
||||
swift,
|
||||
JINJA_SNIPPET_CONFIG, {'role': 'CustomRole'},
|
||||
'customrole-config.yaml',
|
||||
constants.DEFAULT_CONTAINER_NAME)
|
||||
|
||||
result = swift.put_object._mock_mock_calls[0]
|
||||
|
||||
self.assertTrue("CustomRole" in str(result))
|
||||
|
||||
def test_j2_render_and_put_include(self):
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == 'foo.yaml':
|
||||
return ['', JINJA_SNIPPET_CONFIG]
|
||||
|
||||
def return_container_files(*args):
|
||||
return ('headers', [{'name': 'foo.yaml'}])
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
swift.get_container = mock.MagicMock(
|
||||
side_effect=return_container_files)
|
||||
|
||||
# Test
|
||||
template_utils.j2_render_and_put(
|
||||
swift,
|
||||
r"{% include 'foo.yaml' %}",
|
||||
{'role': 'CustomRole'},
|
||||
'customrole-config.yaml',
|
||||
constants.DEFAULT_CONTAINER_NAME)
|
||||
|
||||
result = swift.put_object._mock_mock_calls[0]
|
||||
|
||||
self.assertTrue("CustomRole" in str(result))
|
||||
|
||||
def test_j2_render_and_put_include_relative(self):
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == 'bar/foo.yaml':
|
||||
return ['', JINJA_SNIPPET_CONFIG]
|
||||
|
||||
def return_container_files(*args):
|
||||
return ('headers', [{'name': 'bar/foo.yaml'}])
|
||||
|
||||
# setup swift
|
||||
swift = mock.MagicMock()
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
swift.get_container = mock.MagicMock(
|
||||
side_effect=return_container_files)
|
||||
|
||||
# Test
|
||||
template_utils.j2_render_and_put(
|
||||
swift,
|
||||
r"{% include 'bar/foo.yaml' %}",
|
||||
{'role': 'CustomRole'},
|
||||
'bar/customrole-config.yaml',
|
||||
constants.DEFAULT_CONTAINER_NAME)
|
||||
|
||||
result = swift.put_object._mock_mock_calls[0]
|
||||
|
||||
self.assertTrue("CustomRole" in str(result))
|
||||
|
||||
def test_get_j2_excludes_file(self):
|
||||
|
||||
swift = mock.MagicMock()
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES]
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
# Test - J2 exclude file with valid templates
|
||||
self.assertTrue({'name': ['puppet/controller-role.yaml']} ==
|
||||
template_utils.get_j2_excludes_file(swift))
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES_EMPTY_LIST]
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
# Test - J2 exclude file with no template to exlude
|
||||
self.assertTrue(
|
||||
{'name': []} == template_utils.get_j2_excludes_file(swift))
|
||||
|
||||
def return_multiple_files(*args):
|
||||
if args[1] == constants.OVERCLOUD_J2_EXCLUDES:
|
||||
return ['', J2_EXCLUDES_EMPTY_FILE]
|
||||
swift.get_object = mock.MagicMock(side_effect=return_multiple_files)
|
||||
# Test - J2 exclude file empty
|
||||
self.assertTrue(
|
||||
{'name': []} == template_utils.get_j2_excludes_file(swift))
|
||||
|
||||
def test_heat_resource_exists(self):
|
||||
heat_client = mock.MagicMock()
|
||||
stack = mock.MagicMock(stack_name='overcloud')
|
||||
heat_client.resources.get.return_value = mock.MagicMock(
|
||||
links=[{'rel': 'stack',
|
||||
'href': 'http://192.0.2.1:8004/v1/'
|
||||
'a959ac7d6a4a475daf2428df315c41ef/'
|
||||
'stacks/overcloud/123'}],
|
||||
logical_resource_id='logical_id',
|
||||
physical_resource_id='resource_id',
|
||||
resource_type='OS::Heat::ResourceGroup',
|
||||
resource_name='InternalApiNetwork'
|
||||
)
|
||||
self.assertTrue(
|
||||
template_utils.heat_resource_exists(
|
||||
heat_client, stack, 'Networks', 'InternalNetwork'))
|
||||
|
||||
def test_no_heat_resource_exists(self):
|
||||
heat_client = mock.MagicMock()
|
||||
stack = mock.MagicMock(stack_name='overcloud')
|
||||
|
||||
def return_not_found(*args):
|
||||
raise heat_exc.HTTPNotFound()
|
||||
|
||||
heat_client.resources.get.side_effect = return_not_found
|
||||
self.assertFalse(
|
||||
template_utils.heat_resource_exists(
|
||||
heat_client, stack, 'Networks', 'InternalNetwork'))
|
||||
|
||||
@mock.patch('tripleo_common.utils.template.heat_resource_exists')
|
||||
@mock.patch('tripleo_common.utils.template.j2_render_and_put')
|
||||
def test_legacy_api_network_exists(self, j2_mock,
|
||||
resource_exists_mock):
|
||||
resource_exists_mock.return_value = True
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'role-networks.role.j2.yaml', JINJA_SNIPPET_ROLE_NETWORKS,
|
||||
ROLE_DATA_ENABLE_NETWORKS)
|
||||
heat = mock.MagicMock()
|
||||
|
||||
# Test
|
||||
template_utils.process_custom_roles(swift, heat)
|
||||
expected_j2_template = swift.get_object(
|
||||
constants.DEFAULT_CONTAINER_NAME, 'network_data.yaml')[1]
|
||||
expected_j2_data = {'roles': [{'name': 'CustomRole'}],
|
||||
'networks': [{'name': 'InternalApi',
|
||||
'compat_name': 'Internal'}]
|
||||
}
|
||||
assert j2_mock.called_with(expected_j2_template, expected_j2_data,
|
||||
'foo.yaml')
|
||||
|
||||
@mock.patch('tripleo_common.utils.template.heat_resource_exists')
|
||||
@mock.patch('tripleo_common.utils.template.j2_render_and_put')
|
||||
def test_no_legacy_api_network_exists(self, j2_mock,
|
||||
resource_exists_mock):
|
||||
resource_exists_mock.return_value = False
|
||||
swift = self._custom_roles_mock_objclient(
|
||||
'role-networks.role.j2.yaml', JINJA_SNIPPET_ROLE_NETWORKS,
|
||||
ROLE_DATA_ENABLE_NETWORKS)
|
||||
heat = mock.MagicMock()
|
||||
|
||||
# Test
|
||||
template_utils.process_custom_roles(swift, heat)
|
||||
expected_j2_template = swift.get_object(
|
||||
constants.DEFAULT_CONTAINER_NAME, 'network_data.yaml')[1]
|
||||
expected_j2_data = {'roles': [{'name': 'CustomRole'}],
|
||||
'networks': [{'name': 'InternalApi'}]
|
||||
}
|
||||
assert j2_mock.called_with(expected_j2_template, expected_j2_data,
|
||||
'foo.yaml')
|
|
@ -0,0 +1,355 @@
|
|||
# Copyright 2016 Red Hat, Inc.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import jinja2
|
||||
import logging
|
||||
import os
|
||||
import six
|
||||
import yaml
|
||||
|
||||
from heatclient import exc as heat_exc
|
||||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
from tripleo_common import constants
|
||||
from tripleo_common.utils import parameters
|
||||
from tripleo_common.utils import plan as plan_utils
|
||||
from tripleo_common.utils import swift as swiftutils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class J2SwiftLoader(jinja2.BaseLoader):
|
||||
"""Jinja2 loader to fetch included files from swift
|
||||
|
||||
This attempts to fetch a template include file from the given container.
|
||||
An optional search path or list of search paths can be provided. By default
|
||||
only the absolute path relative to the container root is searched.
|
||||
"""
|
||||
|
||||
def __init__(self, swift, container, searchpath=None):
|
||||
self.swift = swift
|
||||
self.container = container
|
||||
if searchpath is not None:
|
||||
if isinstance(searchpath, six.string_types):
|
||||
self.searchpath = [searchpath]
|
||||
else:
|
||||
self.searchpath = list(searchpath)
|
||||
else:
|
||||
self.searchpath = []
|
||||
# Always search the absolute path from the root of the swift container
|
||||
if '' not in self.searchpath:
|
||||
self.searchpath.append('')
|
||||
|
||||
def get_source(self, environment, template):
|
||||
pieces = jinja2.loaders.split_template_path(template)
|
||||
for searchpath in self.searchpath:
|
||||
template_path = os.path.join(searchpath, *pieces)
|
||||
try:
|
||||
source = swiftutils.get_object_string(self.swift,
|
||||
self.container,
|
||||
template_path)
|
||||
return source, None, False
|
||||
except swiftexceptions.ClientException:
|
||||
pass
|
||||
raise jinja2.exceptions.TemplateNotFound(template)
|
||||
|
||||
|
||||
def j2_render_and_put(swift, j2_template, j2_data,
|
||||
container=constants.DEFAULT_CONTAINER_NAME,
|
||||
outfile_name=None):
|
||||
|
||||
yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
|
||||
|
||||
# Search for templates relative to the current template path first
|
||||
template_base = os.path.dirname(yaml_f)
|
||||
j2_loader = J2SwiftLoader(swift, container, template_base)
|
||||
|
||||
try:
|
||||
# Render the j2 template
|
||||
template = jinja2.Environment(loader=j2_loader).from_string(
|
||||
j2_template)
|
||||
r_template = template.render(**j2_data)
|
||||
except jinja2.exceptions.TemplateError as ex:
|
||||
error_msg = ("Error rendering template %s : %s"
|
||||
% (yaml_f, six.text_type(ex)))
|
||||
LOG.error(error_msg)
|
||||
raise RuntimeError(error_msg)
|
||||
|
||||
try:
|
||||
# write the template back to the plan container
|
||||
LOG.info("Writing rendered template %s" % yaml_f)
|
||||
swiftutils.put_object_string(swift, container, yaml_f,
|
||||
r_template)
|
||||
except swiftexceptions.ClientException:
|
||||
error_msg = ("Error storing file %s in container %s"
|
||||
% (yaml_f, container))
|
||||
LOG.error(error_msg)
|
||||
raise RuntimeError(error_msg)
|
||||
|
||||
|
||||
def get_j2_excludes_file(swift, container=constants.DEFAULT_CONTAINER_NAME):
|
||||
try:
|
||||
j2_excl_file = swiftutils.get_object_string(
|
||||
swift, container, constants.OVERCLOUD_J2_EXCLUDES)
|
||||
j2_excl_data = yaml.safe_load(j2_excl_file)
|
||||
if (j2_excl_data is None or j2_excl_data.get('name') is None):
|
||||
j2_excl_data = {"name": []}
|
||||
LOG.info("j2_excludes.yaml is either empty or there are "
|
||||
"no templates to exclude, defaulting the J2 "
|
||||
"excludes list to: %s" % j2_excl_data)
|
||||
except swiftexceptions.ClientException:
|
||||
j2_excl_data = {"name": []}
|
||||
LOG.info("No J2 exclude file found, defaulting "
|
||||
"the J2 excludes list to: %s" % j2_excl_data)
|
||||
return j2_excl_data
|
||||
|
||||
|
||||
def heat_resource_exists(heat, stack, nested_stack_name, resource_name):
|
||||
if stack is None:
|
||||
LOG.debug("Resource does not exist because stack does not exist")
|
||||
return False
|
||||
|
||||
try:
|
||||
nested_stack = heat.resources.get(stack.id, nested_stack_name)
|
||||
except heat_exc.HTTPNotFound:
|
||||
LOG.debug(
|
||||
"Resource does not exist because {} stack does "
|
||||
"not exist".format(nested_stack_name))
|
||||
return False
|
||||
|
||||
try:
|
||||
heat.resources.get(nested_stack.physical_resource_id,
|
||||
resource_name)
|
||||
except heat_exc.HTTPNotFound:
|
||||
LOG.debug("Resource does not exist: {}".format(resource_name))
|
||||
return False
|
||||
else:
|
||||
LOG.debug("Resource exists: {}".format(resource_name))
|
||||
return True
|
||||
|
||||
|
||||
def process_custom_roles(swift, heat,
|
||||
container=constants.DEFAULT_CONTAINER_NAME):
|
||||
try:
|
||||
j2_role_file = swiftutils.get_object_string(
|
||||
swift, container, constants.OVERCLOUD_J2_ROLES_NAME)
|
||||
role_data = yaml.safe_load(j2_role_file)
|
||||
except swiftexceptions.ClientException:
|
||||
LOG.info("No %s file found, skipping jinja templating"
|
||||
% constants.OVERCLOUD_J2_ROLES_NAME)
|
||||
return
|
||||
|
||||
try:
|
||||
j2_network_file = swiftutils.get_object_string(
|
||||
swift, container, constants.OVERCLOUD_J2_NETWORKS_NAME)
|
||||
network_data = yaml.safe_load(j2_network_file)
|
||||
# Allow no networks defined in network_data
|
||||
if network_data is None:
|
||||
network_data = []
|
||||
except swiftexceptions.ClientException:
|
||||
# Until t-h-t contains network_data.yaml we tolerate a missing file
|
||||
LOG.warning("No %s file found, ignoring"
|
||||
% constants.OVERCLOUD_J2_ROLES_NAME)
|
||||
network_data = []
|
||||
|
||||
j2_excl_data = get_j2_excludes_file(swift, container)
|
||||
|
||||
try:
|
||||
# Iterate over all files in the plan container
|
||||
# we j2 render any with the .j2.yaml suffix
|
||||
container_files = swift.get_container(container)
|
||||
except swiftexceptions.ClientException as ex:
|
||||
error_msg = ("Error listing contents of container %s : %s"
|
||||
% (container, six.text_type(ex)))
|
||||
LOG.error(error_msg)
|
||||
raise RuntimeError(error_msg)
|
||||
|
||||
role_names = [r.get('name') for r in role_data]
|
||||
r_map = {}
|
||||
for r in role_data:
|
||||
r_map[r.get('name')] = r
|
||||
excl_templates = j2_excl_data.get('name')
|
||||
|
||||
stack = None
|
||||
try:
|
||||
stack = heat.stacks.get(container, resolve_outputs=False)
|
||||
except heat_exc.HTTPNotFound:
|
||||
LOG.debug("Stack does not exist")
|
||||
|
||||
n_map = {}
|
||||
for n in network_data:
|
||||
if n.get('enabled') is not False:
|
||||
n_map[n.get('name')] = n
|
||||
if not n.get('name_lower'):
|
||||
n_map[n.get('name')]['name_lower'] = n.get('name').lower()
|
||||
if n.get('name') == constants.API_NETWORK and 'compat_name' \
|
||||
not in n.keys():
|
||||
# Check to see if legacy named API network exists
|
||||
# and if so we need to set compat_name
|
||||
api_net = "{}Network".format(constants.LEGACY_API_NETWORK)
|
||||
if heat_resource_exists(heat, stack, 'Networks', api_net):
|
||||
n['compat_name'] = 'Internal'
|
||||
LOG.info("Upgrade compatibility enabled for legacy "
|
||||
"network resource Internal.")
|
||||
else:
|
||||
LOG.info("skipping %s network: network is disabled." %
|
||||
n.get('name'))
|
||||
|
||||
plan_utils.cache_delete(swift, container, "tripleo.parameters.get")
|
||||
|
||||
for f in [f.get('name') for f in container_files[1]]:
|
||||
# We do three templating passes here:
|
||||
# 1. *.role.j2.yaml - we template just the role name
|
||||
# and create multiple files (one per role)
|
||||
# 2 *.network.j2.yaml - we template the network name and
|
||||
# data and create multiple files for networks and
|
||||
# network ports (one per network)
|
||||
# 3. *.j2.yaml - we template with all roles_data,
|
||||
# and create one file common to all roles
|
||||
if f.endswith('.role.j2.yaml'):
|
||||
LOG.info("jinja2 rendering role template %s" % f)
|
||||
j2_template = swiftutils.get_object_string(swift,
|
||||
container, f)
|
||||
LOG.info("jinja2 rendering roles %s" % ","
|
||||
.join(role_names))
|
||||
for role in role_names:
|
||||
LOG.info("jinja2 rendering role %s" % role)
|
||||
out_f = "-".join(
|
||||
[role.lower(),
|
||||
os.path.basename(f).replace('.role.j2.yaml',
|
||||
'.yaml')])
|
||||
out_f_path = os.path.join(os.path.dirname(f), out_f)
|
||||
if ('network/config' in os.path.dirname(f) and
|
||||
r_map[role].get('deprecated_nic_config_name')):
|
||||
d_name = r_map[role].get('deprecated_nic_config_name')
|
||||
out_f_path = os.path.join(os.path.dirname(f), d_name)
|
||||
elif ('network/config' in os.path.dirname(f)):
|
||||
d_name = "%s.yaml" % role.lower()
|
||||
out_f_path = os.path.join(os.path.dirname(f), d_name)
|
||||
if not (out_f_path in excl_templates):
|
||||
if '{{role.name}}' in j2_template:
|
||||
j2_data = {'role': r_map[role],
|
||||
'networks': network_data}
|
||||
j2_render_and_put(swift, j2_template,
|
||||
j2_data, container,
|
||||
out_f_path)
|
||||
else:
|
||||
# Backwards compatibility with templates
|
||||
# that specify {{role}} vs {{role.name}}
|
||||
j2_data = {'role': role, 'networks': network_data}
|
||||
LOG.debug("role legacy path for role %s" % role)
|
||||
j2_render_and_put(swift, j2_template,
|
||||
j2_data, container,
|
||||
out_f_path)
|
||||
else:
|
||||
LOG.info("Skipping rendering of %s, defined in %s" %
|
||||
(out_f_path, j2_excl_data))
|
||||
|
||||
elif (f.endswith('.network.j2.yaml')):
|
||||
LOG.info("jinja2 rendering network template %s" % f)
|
||||
j2_template = swiftutils.get_object_string(swift,
|
||||
container,
|
||||
f)
|
||||
LOG.info("jinja2 rendering networks %s" % ",".join(n_map))
|
||||
for network in n_map:
|
||||
j2_data = {'network': n_map[network]}
|
||||
# Output file names in "<name>.yaml" format
|
||||
out_f = os.path.basename(f).replace('.network.j2.yaml',
|
||||
'.yaml')
|
||||
if os.path.dirname(f).endswith('ports'):
|
||||
out_f = out_f.replace('port',
|
||||
n_map[network]['name_lower'])
|
||||
else:
|
||||
out_f = out_f.replace('network',
|
||||
n_map[network]['name_lower'])
|
||||
out_f_path = os.path.join(os.path.dirname(f), out_f)
|
||||
if not (out_f_path in excl_templates):
|
||||
j2_render_and_put(swift, j2_template,
|
||||
j2_data, container,
|
||||
out_f_path)
|
||||
else:
|
||||
LOG.info("Skipping rendering of %s, defined in %s" %
|
||||
(out_f_path, j2_excl_data))
|
||||
|
||||
elif f.endswith('.j2.yaml'):
|
||||
LOG.info("jinja2 rendering %s" % f)
|
||||
j2_template = swiftutils.get_object_string(swift,
|
||||
container,
|
||||
f)
|
||||
j2_data = {'roles': role_data, 'networks': network_data}
|
||||
out_f = f.replace('.j2.yaml', '.yaml')
|
||||
j2_render_and_put(swift, j2_template,
|
||||
j2_data, container,
|
||||
out_f)
|
||||
|
||||
|
||||
def process_templates(swift, heat, container=constants.DEFAULT_CONTAINER_NAME):
|
||||
error_text = None
|
||||
try:
|
||||
plan_env = plan_utils.get_env(swift, container)
|
||||
except swiftexceptions.ClientException as err:
|
||||
err_msg = ("Error retrieving environment for plan %s: %s" % (
|
||||
container, err))
|
||||
LOG.exception(err_msg)
|
||||
raise RuntimeError(error_text)
|
||||
|
||||
try:
|
||||
# if the jinja overcloud template exists, process it and write it
|
||||
# back to the swift container before continuing processing. The
|
||||
# method called below should handle the case where the files are
|
||||
# not found in swift, but if they are found and an exception
|
||||
# occurs during processing, then it will be raised.
|
||||
process_custom_roles(swift, heat, container)
|
||||
except Exception as err:
|
||||
LOG.exception("Error occurred while processing custom roles.")
|
||||
raise RuntimeError(six.text_type(err))
|
||||
|
||||
template_name = plan_env.get('template', "")
|
||||
|
||||
template_object = os.path.join(swift.url, container,
|
||||
template_name)
|
||||
LOG.debug('Template: %s' % template_name)
|
||||
try:
|
||||
template_files, template = plan_utils.get_template_contents(
|
||||
swift, template_object)
|
||||
except Exception as err:
|
||||
error_text = six.text_type(err)
|
||||
LOG.exception("Error occurred while fetching %s" % template_object)
|
||||
|
||||
temp_env_paths = []
|
||||
try:
|
||||
env_paths, temp_env_paths = plan_utils.build_env_paths(
|
||||
swift, container, plan_env)
|
||||
env_files, env = plan_utils.process_environments_and_files(
|
||||
swift, env_paths)
|
||||
parameters.convert_docker_params(env)
|
||||
except Exception as err:
|
||||
error_text = six.text_type(err)
|
||||
LOG.exception("Error occurred while processing plan files.")
|
||||
finally:
|
||||
# cleanup any local temp files
|
||||
for f in temp_env_paths:
|
||||
os.remove(f)
|
||||
|
||||
if error_text:
|
||||
raise RuntimeError(six.text_type(error_text))
|
||||
|
||||
files = dict(list(template_files.items()) + list(env_files.items()))
|
||||
|
||||
return {
|
||||
'stack_name': container,
|
||||
'template': template,
|
||||
'environment': env,
|
||||
'files': files
|
||||
}
|
|
@ -126,9 +126,16 @@ workflows:
|
|||
|
||||
ensure_passwords_exist:
|
||||
action: tripleo.parameters.generate_passwords container=<% $.container %>
|
||||
on-success: add_root_stack_name
|
||||
on-success: container_images_prepare
|
||||
on-error: ensure_passwords_exist_set_status_failed
|
||||
|
||||
container_images_prepare:
|
||||
description: >
|
||||
Populate all container image parameters with default values.
|
||||
action: tripleo.container_images.prepare container=<% $.container %>
|
||||
on-success: add_root_stack_name
|
||||
on-error: container_images_prepare_set_status_failed
|
||||
|
||||
add_root_stack_name:
|
||||
action: tripleo.parameters.update
|
||||
input:
|
||||
|
@ -136,24 +143,12 @@ workflows:
|
|||
validate: <% $.validate_stack %>
|
||||
parameters:
|
||||
RootStackName: <% $.container %>
|
||||
on-success: container_images_prepare
|
||||
on-success: set_status_success
|
||||
publish-on-error:
|
||||
status: FAILED
|
||||
message: <% task().result %>
|
||||
on-error: send_message
|
||||
|
||||
container_images_prepare:
|
||||
description: >
|
||||
Populate all container image parameters with default values.
|
||||
action: tripleo.container_images.prepare container=<% $.container %>
|
||||
on-success: process_templates
|
||||
on-error: container_images_prepare_set_status_failed
|
||||
|
||||
process_templates:
|
||||
action: tripleo.templates.process container=<% $.container %>
|
||||
on-success: set_status_success
|
||||
on-error: process_templates_set_status_failed
|
||||
|
||||
set_status_success:
|
||||
on-success: send_message
|
||||
publish:
|
||||
|
@ -184,12 +179,6 @@ workflows:
|
|||
status: FAILED
|
||||
message: <% task(ensure_passwords_exist).result %>
|
||||
|
||||
process_templates_set_status_failed:
|
||||
on-success: send_message
|
||||
publish:
|
||||
status: FAILED
|
||||
message: <% task(process_templates).result %>
|
||||
|
||||
container_images_prepare_set_status_failed:
|
||||
on-success: send_message
|
||||
publish:
|
||||
|
@ -280,9 +269,16 @@ workflows:
|
|||
|
||||
ensure_passwords_exist:
|
||||
action: tripleo.parameters.generate_passwords container=<% $.container %>
|
||||
on-success: add_root_stack_name
|
||||
on-success: container_images_prepare
|
||||
on-error: ensure_passwords_exist_set_status_failed
|
||||
|
||||
container_images_prepare:
|
||||
description: >
|
||||
Populate all container image parameters with default values.
|
||||
action: tripleo.container_images.prepare container=<% $.container %>
|
||||
on-success: add_root_stack_name
|
||||
on-error: container_images_prepare_set_status_failed
|
||||
|
||||
add_root_stack_name:
|
||||
action: tripleo.parameters.update
|
||||
input:
|
||||
|
@ -290,19 +286,12 @@ workflows:
|
|||
validate: <% $.validate_stack %>
|
||||
parameters:
|
||||
RootStackName: <% $.container %>
|
||||
on-success: container_images_prepare
|
||||
on-success: create_swift_rings_backup_plan
|
||||
publish-on-error:
|
||||
status: FAILED
|
||||
message: <% task().result %>
|
||||
on-error: send_message
|
||||
|
||||
container_images_prepare:
|
||||
description: >
|
||||
Populate all container image parameters with default values.
|
||||
action: tripleo.container_images.prepare container=<% $.container %>
|
||||
on-success: create_swift_rings_backup_plan
|
||||
on-error: container_images_prepare_set_status_failed
|
||||
|
||||
create_swift_rings_backup_plan:
|
||||
workflow: tripleo.swift_backup.v1.create_swift_backup_container_plan
|
||||
on-success: create_ceph_ansible_fetch_directory_backup_plan
|
||||
|
@ -333,7 +322,9 @@ workflows:
|
|||
|
||||
does_ceph_ansible_fetch_directory_backup_need_rename:
|
||||
workflow: tripleo.rename_ceph_ansible_fetch_directory.v1.check_and_rename
|
||||
on-success: process_templates
|
||||
on-success:
|
||||
- set_status_success: <% $.plan_environment = null %>
|
||||
- upload_plan_environment: <% $.plan_environment != null %>
|
||||
on-error: does_ceph_ansible_fetch_directory_backup_need_rename_set_status_failed
|
||||
input:
|
||||
container: <% $.container %>
|
||||
|
@ -341,17 +332,10 @@ workflows:
|
|||
container_suffix: "_ceph_ansible_fetch_dir"
|
||||
swift_tar: "temporary_dir.tar.gz"
|
||||
|
||||
process_templates:
|
||||
action: tripleo.templates.process container=<% $.container %>
|
||||
on-success:
|
||||
- set_status_success: <% $.plan_environment = null %>
|
||||
- upload_plan_environment: <% $.plan_environment != null %>
|
||||
on-error: process_templates_set_status_failed
|
||||
|
||||
upload_plan_environment:
|
||||
action: tripleo.templates.upload_plan_environment container=<% $.container %> plan_environment=<% $.plan_environment %>
|
||||
on-success: set_status_success
|
||||
on-error: process_templates_set_status_failed
|
||||
on-error: upload_plan_set_status_failed
|
||||
|
||||
set_status_success:
|
||||
on-success: send_message
|
||||
|
@ -389,11 +373,11 @@ workflows:
|
|||
status: FAILED
|
||||
message: <% task(upload_templates_directory).result %>
|
||||
|
||||
process_templates_set_status_failed:
|
||||
upload_plan_set_status_failed:
|
||||
on-success: send_message
|
||||
publish:
|
||||
status: FAILED
|
||||
message: <% task(process_templates).result %>
|
||||
message: <% task(upload_plan_environment).result %>
|
||||
|
||||
ensure_passwords_exist_set_status_failed:
|
||||
on-success: send_message
|
||||
|
|
Loading…
Reference in New Issue