openstack
/
tripleo-common
Code Issues Proposed changes

Adds action for template processing 

This patch moves the code that deals with calling out to
heatclient to process multiple files and environments from
utils/templates.py into an action that will provide a base object
for the deployment and parameters actions to inherit from. This
patch also removes the resulting dead code (utils/templates.py)
and its associated test.

Co-Authored-By: Dougal Matthews <dougal@redhat.com>
Change-Id: I8efb53ac14c3b952743d1871df8d63d32685b6b1
This commit is contained in:
Ryan Brady 2016-07-05 07:06:37 -04:00
parent 5be0dd82fe
commit 8f2ee4a380
6 changed files with 151 additions and 210 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
setup.cfg
View File

@ -57,6 +57,7 @@ output_file = tripleo_common/locale/tripleo-common.pot
[entry_points]
mistral.actions =
tripleo.upload_default_templates = tripleo_common.actions.templates:UploadTemplatesAction
tripleo.process_templates = tripleo_common.actions.templates:ProcessTemplatesAction
tripleo.create_container = tripleo_common.actions.plan:CreateContainerAction
tripleo.create_plan = tripleo_common.actions.plan:CreatePlanAction
tripleo.deploy_config = tripleo_common.actions.deployment:OrchestrationDeployAction

3
tripleo_common/actions/heat_capabilities.py
View File

@ -130,8 +130,7 @@ class UpdateCapabilitiesAction(base.TripleOAction):
mistral_client.environments.update(**env_kwargs)
except Exception as mistral_err:
err_msg = (
"Error retrieving mistral "
"environment. %s" % mistral_err)
"Error retrieving mistral environment. %s" % mistral_err)
LOG.exception(err_msg)
return mistral_workflow_utils.Result(
None,

102
tripleo_common/actions/templates.py
View File

@ -12,9 +12,16 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import logging
import os
import requests
import tempfile as tf
from heatclient.common import template_utils
from mistral import context
from mistral.workflow import utils as mistral_workflow_utils
from tripleo_common.actions import base
from tripleo_common import constants
from tripleo_common.utils import tarball
@ -22,6 +29,14 @@ from tripleo_common.utils import tarball
LOG = logging.getLogger(__name__)
def _create_temp_file(data):
handle, env_temp_file = tf.mkstemp()
with open(env_temp_file, 'w') as temp_file:
temp_file.write(json.dumps(data))
os.close(handle)
return env_temp_file
class UploadTemplatesAction(base.TripleOAction):
"""Upload default heat templates for TripleO.
@ -38,3 +53,90 @@ class UploadTemplatesAction(base.TripleOAction):
self._get_object_client(),
tmp_tarball.name,
self.container)
class ProcessTemplatesAction(base.TripleOAction):
def __init__(self, container=constants.DEFAULT_CONTAINER_NAME):
super(ProcessTemplatesAction, self).__init__()
self.container = container
def run(self):
error_text = None
ctx = context.ctx()
swift = self._get_object_client()
mistral = self._get_workflow_client()
try:
mistral_environment = mistral.environments.get(self.container)
except Exception as mistral_err:
error_text = mistral_err.message
LOG.exception(
"Error retrieving Mistral Environment: %s" % self.container)
return mistral_workflow_utils.Result(error=error_text)
template_name = mistral_environment.variables.get('template')
environments = mistral_environment.variables.get('environments')
env_paths = []
temp_files = []
template_object = os.path.join(swift.url, self.container,
template_name)
LOG.debug('Template: %s' % template_name)
LOG.debug('Environments: %s' % environments)
try:
for env in environments:
if env.get('path'):
env_paths.append(os.path.join(swift.url, self.container,
env['path']))
elif env.get('data'):
env_temp_file = _create_temp_file(env['data'])
temp_files.append(env_temp_file)
env_paths.append(env_temp_file)
# handle user set parameter values
params = mistral_environment.variables.get('parameter_defaults')
if params:
env_temp_file = _create_temp_file(
{'parameter_defaults': params})
temp_files.append(env_temp_file)
env_paths.append(env_temp_file)
def _env_path_is_object(env_path):
retval = env_path.startswith(swift.url)
LOG.debug('_env_path_is_object %s: %s' % (env_path, retval))
return retval
def _object_request(method, url, token=ctx.auth_token):
return requests.request(
method, url, headers={'X-Auth-Token': token}).content
template_files, template = template_utils.get_template_contents(
template_object=template_object,
object_request=_object_request)
env_files, env = (
template_utils.process_multiple_environments_and_files(
env_paths=env_paths,
env_path_is_object=_env_path_is_object,
object_request=_object_request))
except Exception as err:
error_text = str(err)
LOG.exception("Error occurred while processing plan files.")
finally:
# cleanup any local temp files
for f in temp_files:
os.remove(f)
if error_text:
return mistral_workflow_utils.Result(error=error_text)
files = dict(list(template_files.items()) + list(env_files.items()))
return {
'stack_name': self.container,
'template': template,
'environment': env,
'files': files
}

47
tripleo_common/tests/actions/test_templates.py
View File

@ -38,3 +38,50 @@ class UploadTemplatesActionTest(base.TestCase):
constants.DEFAULT_TEMPLATES_PATH, 'test')
mock_extract_tar.assert_called_once_with(
mock_get_swift.return_value, 'test', 'tar-container')
class ProcessTemplatesActionTest(base.TestCase):
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('heatclient.common.template_utils.get_template_contents')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'_get_workflow_client')
@mock.patch('tripleo_common.actions.base.TripleOAction._get_object_client')
@mock.patch('mistral.context.ctx')
def test_run(self, mock_ctx, mock_get_object_client,
mock_get_workflow_client, mock_get_template_contents,
mock_process_multiple_environments_and_files):
mock_ctx.return_value = mock.MagicMock()
mock_get_object_client.return_value = mock.MagicMock(
url="http://test.com")
mock_mistral = mock.MagicMock()
mock_env = mock.MagicMock()
mock_env.variables = {
'temp_environment': 'temp_environment',
'template': 'template',
'environments': [{u'path': u'environments/test.yaml'}],
}
mock_mistral.environments.get.return_value = mock_env
mock_get_workflow_client.return_value = mock_mistral
mock_get_template_contents.return_value = ({}, {
'heat_template_version': '2016-04-30'
})
mock_process_multiple_environments_and_files.return_value = ({}, {})
# Test
action = templates.ProcessTemplatesAction()
result = action.run()
# Verify the values we get out
self.assertEqual(result, {
'environment': {},
'files': {},
'stack_name': constants.DEFAULT_CONTAINER_NAME,
'template': {
'heat_template_version': '2016-04-30'
}
})

86
tripleo_common/tests/utils/test_templates.py
View File

@ -1,86 +0,0 @@
# Copyright 2015 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
from tripleo_common.tests import base
from tripleo_common.utils import templates
PLAN_DATA = {
'/path/to/overcloud.yaml': {
'contents': 'heat_template_version: 2015-04-30',
'meta': {'file-type': 'root-template'},
},
'/path/to/environment.yaml': {
'contents': "parameters:\n"
" one: uno\n"
" obj:\n"
" two: due\n"
" three: tre\n",
'meta': {
'file-type': 'root-environment',
'enabled': 'True'
}
},
'/path/to/network-isolation.json': {
'contents': '{"parameters": {"one": "one"}}',
'meta': {'file-type': 'environment'},
},
'/path/to/ceph-storage-env.yaml': {
'contents': "parameters:\n"
" obj:\n"
" two: dos,\n"
" three: three",
'meta': {'file-type': 'environment'},
},
'/path/to/poc-custom-env.yaml': {
'contents': "parameters:\n"
" obj:\n"
" two: two\n"
" some::resource: /path/to/somefile.yaml",
'meta': {'file-type': 'environment'}
},
'/path/to/somefile.yaml': {'contents': "description: lorem ipsum"}
}
class UtilsTemplatesTest(base.TestCase):
def setUp(self):
super(UtilsTemplatesTest, self).setUp()
@mock.patch("requests.request")
def test_preprocess_templates(self, mock_request):
# Setup
envs = []
mock_request.return_value = mock.Mock(content="""{
"heat_template_version": "2016-04-08"
}""")
# Test a basic call to check the main code paths
result = templates.preprocess_templates(
"swift_base_url", "container", "template", envs, "auth_token")
# Verify the values we get out
self.assertEqual(result, {
'environment': {},
'files': {},
'stack_name': 'container',
'template': {
'heat_template_version': '2016-04-08'
}
})

122
tripleo_common/utils/templates.py
View File

@ -1,122 +0,0 @@
# Copyright 2015 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import logging
import os
import requests
import tempfile
import yaml
from heatclient.common import template_utils
LOG = logging.getLogger(__name__)
def _get_dict_from_env_string(env_name, env_string):
"""Returns environment dict, either from yaml or json."""
if '.yaml' in env_name:
return yaml.load(env_string)
else:
return json.loads(env_string)
def deep_update(base, new):
"""Updates a given dictionary with a nested dictionary of varying depth
:param base: The dictionary to update
:param new: The dictionary to merge into the base dictionary
:return: a combined nested dictionary
"""
for key, val in new.items():
if isinstance(val, dict):
tmp = deep_update(base.get(key, {}), val)
base[key] = tmp
else:
base[key] = val
return base
def preprocess_templates(swift_base_url, container_name, template,
environments, auth_token):
"""Pre-processes and organizes plan files
This method processes heat templates and environments by collecting the
remote paths of the files in a given swift container and combining them
with given environment data and uses methods in python-heatclient to get
template contents and process the files with respect to order. This
method also sets the stack_name returned in the results to the same name
as the given container.
:param swift_base_url: the endpoint url for swift
:param container_name: name of the swift container that holds heat
templates for a deployment plan
:param template: the root template of a given plan
:param environments: environment files or yaml contents to be combined
:param auth_token: keystone authentication token for accessing heat and
swift to retrieve file contents.
:return: dict of heat stack name, template, combined environment and files
"""
template_object = os.path.join(swift_base_url, container_name, template)
env_paths = []
temp_files = []
LOG.debug('Template: %s' % template)
LOG.debug('Environments: %s' % environments)
try:
for env in environments:
if env.get('path'):
env_paths.append(os.path.join(swift_base_url, container_name,
env['path']))
elif env.get('data'):
handle, env_temp_file = tempfile.mkstemp()
with open(env_temp_file, 'w') as temp_file:
temp_file.write(json.dumps(env['data']))
os.close(handle)
temp_files.append(env_temp_file)
env_paths.append(env_temp_file)
def _env_path_is_object(env_path):
if env_path in temp_files:
LOG.debug('_env_path_is_object %s: False' % env_path)
return False
else:
LOG.debug('_env_path_is_object %s: True' % env_path)
return True
def _object_request(method, url, token=auth_token):
return requests.request(method, url,
headers={'X-Auth-Token': token}).content
template_files, template = template_utils.get_template_contents(
template_object=template_object,
object_request=_object_request)
env_files, env = (
template_utils.process_multiple_environments_and_files(
env_paths=env_paths,
env_path_is_object=_env_path_is_object,
object_request=_object_request))
finally:
# cleanup any local temp files
for f in temp_files:
os.remove(f)
files = dict(list(template_files.items()) + list(env_files.items()))
return {
'stack_name': container_name,
'template': template,
'environment': env,
'files': files
}