Refactor functions out of ProcessTemplateAction
This needs to be common utility code for the next change in this series, and breaking them out allows the test coverage to be improved. Change-Id: I9d55ccfbd2875046dc7512413c3be3254a45910a Blueprint: container-prepare-workflow
This commit is contained in:
Steve Baker
parent
5c0fc20c2a
commit
5c77d51678
@ -13,15 +13,12 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import jinja2
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import six
|
||||
import tempfile as tf
|
||||
import yaml
|
||||
|
||||
from heatclient.common import template_utils
|
||||
from heatclient import exc as heat_exc
|
||||
from mistral_lib import actions
|
||||
from swiftclient import exceptions as swiftexceptions
|
||||
@ -34,14 +31,6 @@ from tripleo_common.utils import tarball
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _create_temp_file(data):
|
||||
handle, env_temp_file = tf.mkstemp()
|
||||
with open(env_temp_file, 'w') as temp_file:
|
||||
temp_file.write(json.dumps(data))
|
||||
os.close(handle)
|
||||
return env_temp_file
|
||||
|
||||
|
||||
class J2SwiftLoader(jinja2.BaseLoader):
|
||||
"""Jinja2 loader to fetch included files from swift
|
||||
|
||||
@ -371,82 +360,29 @@ class ProcessTemplatesAction(base.TripleOAction):
|
||||
return actions.Result(error=six.text_type(err))
|
||||
|
||||
template_name = plan_env.get('template', "")
|
||||
environments = plan_env.get('environments', [])
|
||||
env_paths = []
|
||||
temp_files = []
|
||||
|
||||
template_object = os.path.join(swift.url, self.container,
|
||||
template_name)
|
||||
|
||||
LOG.debug('Template: %s' % template_name)
|
||||
LOG.debug('Environments: %s' % environments)
|
||||
try:
|
||||
for env in environments:
|
||||
if env.get('path'):
|
||||
env_paths.append(os.path.join(swift.url, self.container,
|
||||
env['path']))
|
||||
elif env.get('data'):
|
||||
env_temp_file = _create_temp_file(env['data'])
|
||||
temp_files.append(env_temp_file)
|
||||
env_paths.append(env_temp_file)
|
||||
|
||||
# create a dict to hold all user set params and merge
|
||||
# them in the appropriate order
|
||||
merged_params = {}
|
||||
# merge generated passwords into params first
|
||||
passwords = plan_env.get('passwords', {})
|
||||
merged_params.update(passwords)
|
||||
|
||||
# derived parameters are merged before 'parameter defaults'
|
||||
# so that user-specified values can override the derived values.
|
||||
derived_params = plan_env.get('derived_parameters', {})
|
||||
merged_params.update(derived_params)
|
||||
|
||||
# handle user set parameter values next in case a user has set
|
||||
# a new value for a password parameter
|
||||
params = plan_env.get('parameter_defaults', {})
|
||||
merged_params = template_utils.deep_update(merged_params, params)
|
||||
|
||||
if merged_params:
|
||||
env_temp_file = _create_temp_file(
|
||||
{'parameter_defaults': merged_params})
|
||||
temp_files.append(env_temp_file)
|
||||
env_paths.append(env_temp_file)
|
||||
|
||||
registry = plan_env.get('resource_registry', {})
|
||||
if registry:
|
||||
env_temp_file = _create_temp_file(
|
||||
{'resource_registry': registry})
|
||||
temp_files.append(env_temp_file)
|
||||
env_paths.append(env_temp_file)
|
||||
|
||||
def _env_path_is_object(env_path):
|
||||
retval = env_path.startswith(swift.url)
|
||||
LOG.debug('_env_path_is_object %s: %s' % (env_path, retval))
|
||||
return retval
|
||||
|
||||
def _object_request(method, url, token=context.auth_token):
|
||||
response = requests.request(
|
||||
method, url, headers={'X-Auth-Token': token})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
template_files, template = template_utils.get_template_contents(
|
||||
template_object=template_object,
|
||||
object_request=_object_request)
|
||||
|
||||
env_files, env = (
|
||||
template_utils.process_multiple_environments_and_files(
|
||||
env_paths=env_paths,
|
||||
env_path_is_object=_env_path_is_object,
|
||||
object_request=_object_request))
|
||||
template_files, template = plan_utils.get_template_contents(
|
||||
swift, template_object)
|
||||
except Exception as err:
|
||||
error_text = six.text_type(err)
|
||||
LOG.exception("Error occurred while fetching %s" % template_object)
|
||||
|
||||
temp_env_paths = []
|
||||
try:
|
||||
env_paths, temp_env_paths = plan_utils.build_env_paths(
|
||||
swift, self.container, plan_env)
|
||||
env_files, env = plan_utils.process_environments_and_files(
|
||||
swift, env_paths)
|
||||
except Exception as err:
|
||||
error_text = six.text_type(err)
|
||||
LOG.exception("Error occurred while processing plan files.")
|
||||
finally:
|
||||
# cleanup any local temp files
|
||||
for f in temp_files:
|
||||
for f in temp_env_paths:
|
||||
os.remove(f)
|
||||
|
||||
if error_text:
|
||||
|
||||
@ -13,7 +13,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import mock
|
||||
import os
|
||||
|
||||
from swiftclient import exceptions as swiftexceptions
|
||||
|
||||
@ -111,3 +113,137 @@ class PlanTest(base.TestCase):
|
||||
|
||||
self.swift.get_object.assert_called()
|
||||
self.swift.put_object.assert_called()
|
||||
|
||||
def test_write_json_temp_file(self):
|
||||
name = plan_utils.write_json_temp_file({'foo': 'bar'})
|
||||
with open(name) as f:
|
||||
self.assertEqual({'foo': 'bar'}, json.load(f))
|
||||
os.remove(name)
|
||||
|
||||
@mock.patch('requests.request', autospec=True)
|
||||
def test_object_request(self, request):
|
||||
request.return_value.content = 'foo'
|
||||
|
||||
content = plan_utils.object_request('GET', '/foo/bar', 'asdf1234')
|
||||
|
||||
self.assertEqual('foo', content)
|
||||
request.assert_called_once_with(
|
||||
'GET', '/foo/bar', headers={'X-Auth-Token': 'asdf1234'})
|
||||
|
||||
@mock.patch('tripleo_common.utils.plan.object_request',
|
||||
autospec=True)
|
||||
def test_process_environments_and_files(self, object_request):
|
||||
swift_url = 'https://192.0.2.1:8443/foo'
|
||||
url = '%s/bar' % swift_url
|
||||
object_request.return_value = 'parameter_defaults: {foo: bar}'
|
||||
swift = mock.Mock()
|
||||
swift.url = swift_url
|
||||
swift.token = 'asdf1234'
|
||||
|
||||
result = plan_utils.process_environments_and_files(swift, [url])
|
||||
|
||||
self.assertEqual(
|
||||
{'parameter_defaults': {'foo': 'bar'}},
|
||||
result[1]
|
||||
)
|
||||
object_request.assert_called_once_with(
|
||||
'GET',
|
||||
'https://192.0.2.1:8443/foo/bar',
|
||||
'asdf1234'
|
||||
)
|
||||
|
||||
@mock.patch('tripleo_common.utils.plan.object_request',
|
||||
autospec=True)
|
||||
def test_get_template_contents(self, object_request):
|
||||
swift_url = 'https://192.0.2.1:8443/foo'
|
||||
url = '%s/bar' % swift_url
|
||||
object_request.return_value = 'heat_template_version: 2016-04-30'
|
||||
swift = mock.Mock()
|
||||
swift.url = swift_url
|
||||
swift.token = 'asdf1234'
|
||||
|
||||
result = plan_utils.get_template_contents(swift, url)
|
||||
|
||||
self.assertEqual(
|
||||
{'heat_template_version': '2016-04-30'},
|
||||
result[1]
|
||||
)
|
||||
object_request.assert_called_once_with(
|
||||
'GET',
|
||||
'https://192.0.2.1:8443/foo/bar',
|
||||
'asdf1234'
|
||||
)
|
||||
|
||||
def test_build_env_paths(self):
|
||||
swift = mock.Mock()
|
||||
swift.url = 'https://192.0.2.1:8443/foo'
|
||||
swift.token = 'asdf1234'
|
||||
plan = {
|
||||
'version': '1.0',
|
||||
'environments': [
|
||||
{'path': 'bar.yaml'},
|
||||
{'data': {
|
||||
'parameter_defaults': {'InlineParam': 1}}}
|
||||
],
|
||||
'passwords': {
|
||||
'ThePassword': 'password1'
|
||||
},
|
||||
'derived_parameters': {
|
||||
'DerivedParam': 'DerivedValue',
|
||||
'MergableParam': {
|
||||
'one': 'derived one',
|
||||
'two': 'derived two',
|
||||
},
|
||||
},
|
||||
'parameter_defaults': {
|
||||
'Foo': 'bar',
|
||||
'MergableParam': {
|
||||
'one': 'user one',
|
||||
'three': 'user three',
|
||||
},
|
||||
},
|
||||
'resource_registry': {
|
||||
'Foo::Bar': 'foo_bar.yaml'
|
||||
},
|
||||
}
|
||||
|
||||
env_paths, temp_env_paths = plan_utils.build_env_paths(
|
||||
swift, 'overcloud', plan)
|
||||
|
||||
self.assertEqual(3, len(temp_env_paths))
|
||||
self.assertEqual(
|
||||
['https://192.0.2.1:8443/foo/overcloud/bar.yaml'] + temp_env_paths,
|
||||
env_paths
|
||||
)
|
||||
|
||||
with open(env_paths[1]) as f:
|
||||
self.assertEqual(
|
||||
{'parameter_defaults': {'InlineParam': 1}},
|
||||
json.load(f)
|
||||
)
|
||||
|
||||
with open(env_paths[2]) as f:
|
||||
self.assertEqual(
|
||||
{'parameter_defaults': {
|
||||
'ThePassword': 'password1',
|
||||
'DerivedParam': 'DerivedValue',
|
||||
'Foo': 'bar',
|
||||
'MergableParam': {
|
||||
'one': 'user one',
|
||||
'two': 'derived two',
|
||||
'three': 'user three',
|
||||
}
|
||||
}},
|
||||
json.load(f)
|
||||
)
|
||||
|
||||
with open(env_paths[3]) as f:
|
||||
self.assertEqual(
|
||||
{'resource_registry': {
|
||||
'Foo::Bar': 'foo_bar.yaml'
|
||||
}},
|
||||
json.load(f)
|
||||
)
|
||||
|
||||
for path in temp_env_paths:
|
||||
os.remove(path)
|
||||
|
||||
@ -13,6 +13,12 @@
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from heatclient.common import template_utils
|
||||
import json
|
||||
import os
|
||||
import requests
|
||||
import tempfile
|
||||
import yaml
|
||||
|
||||
from tripleo_common import constants
|
||||
@ -71,3 +77,86 @@ def put_user_env(swift, container_name, env):
|
||||
constants.USER_ENVIRONMENT,
|
||||
yaml.safe_dump(env, default_flow_style=False)
|
||||
)
|
||||
|
||||
|
||||
def write_json_temp_file(data):
|
||||
"""Writes the provided data to a json file and return the filename"""
|
||||
with tempfile.NamedTemporaryFile(delete=False, mode='w') as temp_file:
|
||||
temp_file.write(json.dumps(data))
|
||||
return temp_file.name
|
||||
|
||||
|
||||
def object_request(method, url, token):
|
||||
"""Fetch an object with the provided token"""
|
||||
response = requests.request(
|
||||
method, url, headers={'X-Auth-Token': token})
|
||||
response.raise_for_status()
|
||||
return response.content
|
||||
|
||||
|
||||
def process_environments_and_files(swift, env_paths):
|
||||
"""Wrap process_multiple_environments_and_files with swift object fetch"""
|
||||
def _env_path_is_object(env_path):
|
||||
return env_path.startswith(swift.url)
|
||||
|
||||
def _object_request(method, url, token=swift.token):
|
||||
return object_request(method, url, token)
|
||||
|
||||
return template_utils.process_multiple_environments_and_files(
|
||||
env_paths=env_paths,
|
||||
env_path_is_object=_env_path_is_object,
|
||||
object_request=_object_request)
|
||||
|
||||
|
||||
def get_template_contents(swift, template_object):
|
||||
"""Wrap get_template_contents with swift object fetch"""
|
||||
def _object_request(method, url, token=swift.token):
|
||||
return object_request(method, url, token)
|
||||
|
||||
return template_utils.get_template_contents(
|
||||
template_object=template_object,
|
||||
object_request=_object_request)
|
||||
|
||||
|
||||
def build_env_paths(swift, container, plan_env):
|
||||
environments = plan_env.get('environments', [])
|
||||
env_paths = []
|
||||
temp_env_paths = []
|
||||
|
||||
for env in environments:
|
||||
if env.get('path'):
|
||||
env_paths.append(os.path.join(swift.url, container, env['path']))
|
||||
elif env.get('data'):
|
||||
env_file = write_json_temp_file(env['data'])
|
||||
temp_env_paths.append(env_file)
|
||||
|
||||
# create a dict to hold all user set params and merge
|
||||
# them in the appropriate order
|
||||
merged_params = {}
|
||||
# merge generated passwords into params first
|
||||
passwords = plan_env.get('passwords', {})
|
||||
merged_params.update(passwords)
|
||||
|
||||
# derived parameters are merged before 'parameter defaults'
|
||||
# so that user-specified values can override the derived values.
|
||||
derived_params = plan_env.get('derived_parameters', {})
|
||||
merged_params.update(derived_params)
|
||||
|
||||
# handle user set parameter values next in case a user has set
|
||||
# a new value for a password parameter
|
||||
params = plan_env.get('parameter_defaults', {})
|
||||
merged_params = template_utils.deep_update(merged_params, params)
|
||||
|
||||
if merged_params:
|
||||
env_temp_file = write_json_temp_file(
|
||||
{'parameter_defaults': merged_params})
|
||||
temp_env_paths.append(env_temp_file)
|
||||
|
||||
registry = plan_env.get('resource_registry', {})
|
||||
if registry:
|
||||
env_temp_file = write_json_temp_file(
|
||||
{'resource_registry': registry})
|
||||
temp_env_paths.append(env_temp_file)
|
||||
|
||||
env_paths.extend(temp_env_paths)
|
||||
return env_paths, temp_env_paths
|
||||
|
||||
Loading…
Reference in New Issue
Block a user