openstack/python-heatclient
Code Issues Proposed changes

Populate files with content from get_file function calls

Browse Source 
This is the client-side implementation of the get_file HOT
intrinsic function. The data for each resource is recursively
crawled to look for {get_file: <path>} calls. The content for
the URL specified by get_file is fetched and the content stored
in the files field in the API request.

If a relative path is specified, that path is converted to an
absolute URL, and that URL is used as the key for the files
content entry. The template is also modified to replace the
relative path with the absolute URL.

This implements the client-side portion of
blueprint get-file

Change-Id: If1da98578bb2919e3c8325442224376d3fadb0bc
This commit is contained in:
Steve Baker
2014-01-12 12:57:22 +13:00
parent 884d5403bc
commit e86a40e3ba
3 changed files with 194 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
heatclient/common/template_utils.py
View File

@@ -14,6 +14,7 @@
# under the License.
import os
import six
import urllib
from heatclient.common import environment_format
@@ -46,28 +47,60 @@ def get_template_contents(template_file=None, template_url=None,
% template_url)
try:
return template_format.parse(tpl)
template = template_format.parse(tpl)
except ValueError as e:
raise exc.CommandError(
'Error parsing template %s %s' % (template_url, e))
files = {}
tmpl_base_url = base_url_for_url(template_url)
resolve_template_get_files(template, files, tmpl_base_url)
return files, template
def get_file_contents(from_dict, files, base_url=None,
ignore_if=None):
for key, value in iter(from_dict.items()):
if ignore_if and ignore_if(key, value):
continue
if base_url and not base_url.endswith('/'):
base_url = base_url + '/'
def resolve_template_get_files(template, files, template_base_url):
str_url = urlutils.urljoin(base_url, value)
try:
files[str_url] = urlutils.urlopen(str_url).read()
except urlutils.URLError:
raise exc.CommandError('Could not fetch %s from the environment'
% str_url)
from_dict[key] = str_url
def ignore_if(key, value):
if key != 'get_file':
return True
if not isinstance(value, six.string_types):
return True
def recurse_if(value):
return isinstance(value, (dict, list))
get_file_contents(template.get('resources'), files, template_base_url,
ignore_if, recurse_if)
def get_file_contents(from_data, files, base_url=None,
ignore_if=None, recurse_if=None):
if recurse_if and recurse_if(from_data):
if isinstance(from_data, dict):
recurse_data = from_data.itervalues()
else:
recurse_data = from_data
for value in recurse_data:
get_file_contents(value, files, base_url, ignore_if, recurse_if)
if isinstance(from_data, dict):
for key, value in iter(from_data.items()):
if ignore_if and ignore_if(key, value):
continue
if base_url and not base_url.endswith('/'):
base_url = base_url + '/'
str_url = urlutils.urljoin(base_url, value)
try:
files[str_url] = urlutils.urlopen(str_url).read()
except urlutils.URLError:
raise exc.CommandError('Could not fetch contents for %s'
% str_url)
# replace the data value with the normalised absolute URL
from_data[key] = str_url
def base_url_for_url(url):
@@ -83,22 +116,21 @@ def normalise_file_path_to_url(path):
return urlutils.urljoin('file:', urllib.pathname2url(path))
def process_environment_and_files(env_path=None, template_path=None):
def process_environment_and_files(env_path=None, template=None,
template_url=None):
files = {}
env = {}
if not env_path:
return files, env
if env_path:
env_url = normalise_file_path_to_url(env_path)
env_base_url = base_url_for_url(env_url)
raw_env = urlutils.urlopen(env_url).read()
env = environment_format.parse(raw_env)
env_url = normalise_file_path_to_url(env_path)
env_base_url = base_url_for_url(env_url)
raw_env = urlutils.urlopen(env_url).read()
env = environment_format.parse(raw_env)
resolve_environment_urls(
env.get('resource_registry'),
files,
env_base_url)
resolve_environment_urls(
env.get('resource_registry'),
files,
env_base_url)
return files, env

104
heatclient/tests/test_template_utils.py
View File

@@ -269,9 +269,10 @@ class TestGetTemplateContents(testtools.TestCase):
tmpl_file.write(tmpl)
tmpl_file.flush()
tmpl_parsed = template_utils.get_template_contents(
files, tmpl_parsed = template_utils.get_template_contents(
tmpl_file.name)
self.assertEqual({"foo": "bar"}, tmpl_parsed)
self.assertEqual({}, files)
def test_get_template_contents_file_empty(self):
with tempfile.NamedTemporaryFile() as tmpl_file:
@@ -316,8 +317,10 @@ class TestGetTemplateContents(testtools.TestCase):
urlutils.urlopen(url).AndReturn(six.StringIO(tmpl))
self.m.ReplayAll()
tmpl_parsed = template_utils.get_template_contents(template_url=url)
files, tmpl_parsed = template_utils.get_template_contents(
template_url=url)
self.assertEqual({"foo": "bar"}, tmpl_parsed)
self.assertEqual({}, files)
def test_get_template_contents_object(self):
tmpl = '{"foo": "bar"}'
@@ -332,14 +335,109 @@ class TestGetTemplateContents(testtools.TestCase):
self.assertEqual('http://no.where/path/to/a.yaml', object_url)
return tmpl
tmpl_parsed = template_utils.get_template_contents(
files, tmpl_parsed = template_utils.get_template_contents(
template_object=url,
object_request=object_request)
self.assertEqual({"foo": "bar"}, tmpl_parsed)
self.assertEqual({}, files)
self.assertTrue(self.object_requested)
class TestTemplateGetFileFunctions(testtools.TestCase):
hot_template = '''heat_template_version: 2013-05-23
resources:
resource1:
type: type1
properties:
foo: {get_file: foo.yaml}
bar:
get_file:
'http://localhost/bar.yaml'
resource2:
type: type1
properties:
baz:
- {get_file: baz/baz1.yaml}
- {get_file: baz/baz2.yaml}
- {get_file: baz/baz3.yaml}
ignored_list: {get_file: [ignore, me]}
ignored_dict: {get_file: {ignore: me}}
ignored_none: {get_file: }
'''
def setUp(self):
super(TestTemplateGetFileFunctions, self).setUp()
self.m = mox.Mox()
self.addCleanup(self.m.VerifyAll)
self.addCleanup(self.m.UnsetStubs)
def test_hot_template(self):
self.m.StubOutWithMock(urlutils, 'urlopen')
tmpl_file = '/home/my/dir/template.yaml'
url = 'file:///home/my/dir/template.yaml'
urlutils.urlopen(url).AndReturn(
six.StringIO(self.hot_template))
urlutils.urlopen(
'http://localhost/bar.yaml').InAnyOrder().AndReturn(
six.StringIO('bar contents'))
urlutils.urlopen(
'file:///home/my/dir/foo.yaml').InAnyOrder().AndReturn(
six.StringIO('foo contents'))
urlutils.urlopen(
'file:///home/my/dir/baz/baz1.yaml').InAnyOrder().AndReturn(
six.StringIO('baz1 contents'))
urlutils.urlopen(
'file:///home/my/dir/baz/baz2.yaml').InAnyOrder().AndReturn(
six.StringIO('baz2 contents'))
urlutils.urlopen(
'file:///home/my/dir/baz/baz3.yaml').InAnyOrder().AndReturn(
six.StringIO('baz3 contents'))
self.m.ReplayAll()
files, tmpl_parsed = template_utils.get_template_contents(
template_file=tmpl_file)
self.assertEqual({
'http://localhost/bar.yaml': 'bar contents',
'file:///home/my/dir/foo.yaml': 'foo contents',
'file:///home/my/dir/baz/baz1.yaml': 'baz1 contents',
'file:///home/my/dir/baz/baz2.yaml': 'baz2 contents',
'file:///home/my/dir/baz/baz3.yaml': 'baz3 contents',
}, files)
self.assertEqual({
'heat_template_version': '2013-05-23',
'resources': {
'resource1': {
'type': 'type1',
'properties': {
'bar': {'get_file': 'http://localhost/bar.yaml'},
'foo': {'get_file': 'file:///home/my/dir/foo.yaml'},
},
},
'resource2': {
'type': 'type1',
'properties': {
'baz': [
{'get_file': 'file:///home/my/dir/baz/baz1.yaml'},
{'get_file': 'file:///home/my/dir/baz/baz2.yaml'},
{'get_file': 'file:///home/my/dir/baz/baz3.yaml'},
],
'ignored_list': {'get_file': ['ignore', 'me']},
'ignored_dict': {'get_file': {'ignore': 'me'}},
'ignored_none': {'get_file': None},
},
}
}
}, tmpl_parsed)
self.m.VerifyAll()
class TestURLFunctions(testtools.TestCase):
def setUp(self):

61
heatclient/v1/shell.py
View File

@@ -24,7 +24,7 @@ import heatclient.exc as exc
@utils.arg('-f', '--template-file', metavar='<FILE>',
help='Path to the template.')
@utils.arg('-e', '--environment-file', metavar='<FILE>',
@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
help='Path to the environment.')
@utils.arg('-u', '--template-url', metavar='<URL>',
help='URL of template.')
@@ -49,7 +49,7 @@ def do_create(hc, args):
@utils.arg('-f', '--template-file', metavar='<FILE>',
help='Path to the template.')
@utils.arg('-e', '--environment-file', metavar='<FILE>',
@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
help='Path to the environment.')
@utils.arg('-u', '--template-url', metavar='<URL>',
help='URL of template.')
@@ -69,7 +69,12 @@ def do_create(hc, args):
help='Name of the stack to create.')
def do_stack_create(hc, args):
'''Create the stack.'''
files, env = template_utils.process_environment_and_files(
tpl_files, template = template_utils.get_template_contents(
args.template_file,
args.template_url,
args.template_object,
hc.http_client.raw_request)
env_files, env = template_utils.process_environment_and_files(
env_path=args.environment_file)
fields = {
@@ -77,12 +82,8 @@ def do_stack_create(hc, args):
'timeout_mins': args.create_timeout,
'disable_rollback': not(args.enable_rollback),
'parameters': utils.format_parameters(args.parameters),
'template': template_utils.get_template_contents(
args.template_file,
args.template_url,
args.template_object,
hc.http_client.raw_request),
'files': files,
'template': template,
'files': dict(tpl_files.items() + env_files.items()),
'environment': env
}
@@ -171,7 +172,7 @@ def do_stack_show(hc, args):
@utils.arg('-f', '--template-file', metavar='<FILE>',
help='Path to the template.')
@utils.arg('-e', '--environment-file', metavar='<FILE>',
@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
help='Path to the environment.')
@utils.arg('-u', '--template-url', metavar='<URL>',
help='URL of template.')
@@ -191,7 +192,7 @@ def do_update(hc, args):
@utils.arg('-f', '--template-file', metavar='<FILE>',
help='Path to the template.')
@utils.arg('-e', '--environment-file', metavar='<FILE>',
@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
help='Path to the environment.')
@utils.arg('-u', '--template-url', metavar='<URL>',
help='URL of template.')
@@ -206,18 +207,21 @@ def do_update(hc, args):
help='Name or ID of stack to update.')
def do_stack_update(hc, args):
'''Update the stack.'''
files, env = template_utils.process_environment_and_files(
tpl_files, template = template_utils.get_template_contents(
args.template_file,
args.template_url,
args.template_object,
hc.http_client.raw_request)
env_files, env = template_utils.process_environment_and_files(
env_path=args.environment_file)
fields = {
'stack_id': args.id,
'parameters': utils.format_parameters(args.parameters),
'template': template_utils.get_template_contents(
args.template_file,
args.template_url,
args.template_object,
hc.http_client.raw_request),
'files': files,
'template': template,
'files': dict(tpl_files.items() + env_files.items()),
'environment': env
}
@@ -285,7 +289,7 @@ def do_template_show(hc, args):
help='URL of template.')
@utils.arg('-f', '--template-file', metavar='<FILE>',
help='Path to the template.')
@utils.arg('-e', '--environment-file', metavar='<FILE>',
@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
help='Path to the environment.')
@utils.arg('-o', '--template-object', metavar='<URL>',
help='URL to retrieve template object (e.g from swift)')
@@ -303,7 +307,7 @@ def do_validate(hc, args):
help='URL of template.')
@utils.arg('-f', '--template-file', metavar='<FILE>',
help='Path to the template.')
@utils.arg('-e', '--environment-file', metavar='<FILE>',
@utils.arg('-e', '--environment-file', metavar='<FILE or URL>',
help='Path to the environment.')
@utils.arg('-o', '--template-object', metavar='<URL>',
help='URL to retrieve template object (e.g from swift)')
@@ -314,16 +318,19 @@ def do_validate(hc, args):
action='append')
def do_template_validate(hc, args):
'''Validate a template with parameters.'''
files, env = template_utils.process_environment_and_files(
tpl_files, template = template_utils.get_template_contents(
args.template_file,
args.template_url,
args.template_object,
hc.http_client.raw_request)
env_files, env = template_utils.process_environment_and_files(
env_path=args.environment_file)
fields = {
'parameters': utils.format_parameters(args.parameters),
'template': template_utils.get_template_contents(
args.template_file,
args.template_url,
args.template_object,
hc.http_client.raw_request),
'files': files,
'template': template,
'files': dict(tpl_files.items() + env_files.items()),
'environment': env
}