diff --git a/heatclient/common/template_utils.py b/heatclient/common/template_utils.py index bd54c9de..cdc71ccd 100644 --- a/heatclient/common/template_utils.py +++ b/heatclient/common/template_utils.py @@ -14,6 +14,7 @@ # under the License. import os +import six import urllib from heatclient.common import environment_format @@ -46,28 +47,60 @@ def get_template_contents(template_file=None, template_url=None, % template_url) try: - return template_format.parse(tpl) + template = template_format.parse(tpl) except ValueError as e: raise exc.CommandError( 'Error parsing template %s %s' % (template_url, e)) + files = {} + tmpl_base_url = base_url_for_url(template_url) + resolve_template_get_files(template, files, tmpl_base_url) + return files, template -def get_file_contents(from_dict, files, base_url=None, - ignore_if=None): - for key, value in iter(from_dict.items()): - if ignore_if and ignore_if(key, value): - continue - if base_url and not base_url.endswith('/'): - base_url = base_url + '/' +def resolve_template_get_files(template, files, template_base_url): - str_url = urlutils.urljoin(base_url, value) - try: - files[str_url] = urlutils.urlopen(str_url).read() - except urlutils.URLError: - raise exc.CommandError('Could not fetch %s from the environment' - % str_url) - from_dict[key] = str_url + def ignore_if(key, value): + if key != 'get_file': + return True + if not isinstance(value, six.string_types): + return True + + def recurse_if(value): + return isinstance(value, (dict, list)) + + get_file_contents(template.get('resources'), files, template_base_url, + ignore_if, recurse_if) + + +def get_file_contents(from_data, files, base_url=None, + ignore_if=None, recurse_if=None): + + if recurse_if and recurse_if(from_data): + if isinstance(from_data, dict): + recurse_data = from_data.itervalues() + else: + recurse_data = from_data + for value in recurse_data: + get_file_contents(value, files, base_url, ignore_if, recurse_if) + + if isinstance(from_data, dict): + for key, value in iter(from_data.items()): + if ignore_if and ignore_if(key, value): + continue + + if base_url and not base_url.endswith('/'): + base_url = base_url + '/' + + str_url = urlutils.urljoin(base_url, value) + try: + files[str_url] = urlutils.urlopen(str_url).read() + except urlutils.URLError: + raise exc.CommandError('Could not fetch contents for %s' + % str_url) + + # replace the data value with the normalised absolute URL + from_data[key] = str_url def base_url_for_url(url): @@ -83,22 +116,21 @@ def normalise_file_path_to_url(path): return urlutils.urljoin('file:', urllib.pathname2url(path)) -def process_environment_and_files(env_path=None, template_path=None): +def process_environment_and_files(env_path=None, template=None, + template_url=None): files = {} env = {} - if not env_path: - return files, env + if env_path: + env_url = normalise_file_path_to_url(env_path) + env_base_url = base_url_for_url(env_url) + raw_env = urlutils.urlopen(env_url).read() + env = environment_format.parse(raw_env) - env_url = normalise_file_path_to_url(env_path) - env_base_url = base_url_for_url(env_url) - raw_env = urlutils.urlopen(env_url).read() - env = environment_format.parse(raw_env) - - resolve_environment_urls( - env.get('resource_registry'), - files, - env_base_url) + resolve_environment_urls( + env.get('resource_registry'), + files, + env_base_url) return files, env diff --git a/heatclient/tests/test_template_utils.py b/heatclient/tests/test_template_utils.py index 2b9b2ae3..2b2fce1e 100644 --- a/heatclient/tests/test_template_utils.py +++ b/heatclient/tests/test_template_utils.py @@ -269,9 +269,10 @@ class TestGetTemplateContents(testtools.TestCase): tmpl_file.write(tmpl) tmpl_file.flush() - tmpl_parsed = template_utils.get_template_contents( + files, tmpl_parsed = template_utils.get_template_contents( tmpl_file.name) self.assertEqual({"foo": "bar"}, tmpl_parsed) + self.assertEqual({}, files) def test_get_template_contents_file_empty(self): with tempfile.NamedTemporaryFile() as tmpl_file: @@ -316,8 +317,10 @@ class TestGetTemplateContents(testtools.TestCase): urlutils.urlopen(url).AndReturn(six.StringIO(tmpl)) self.m.ReplayAll() - tmpl_parsed = template_utils.get_template_contents(template_url=url) + files, tmpl_parsed = template_utils.get_template_contents( + template_url=url) self.assertEqual({"foo": "bar"}, tmpl_parsed) + self.assertEqual({}, files) def test_get_template_contents_object(self): tmpl = '{"foo": "bar"}' @@ -332,14 +335,109 @@ class TestGetTemplateContents(testtools.TestCase): self.assertEqual('http://no.where/path/to/a.yaml', object_url) return tmpl - tmpl_parsed = template_utils.get_template_contents( + files, tmpl_parsed = template_utils.get_template_contents( template_object=url, object_request=object_request) self.assertEqual({"foo": "bar"}, tmpl_parsed) + self.assertEqual({}, files) self.assertTrue(self.object_requested) +class TestTemplateGetFileFunctions(testtools.TestCase): + + hot_template = '''heat_template_version: 2013-05-23 +resources: + resource1: + type: type1 + properties: + foo: {get_file: foo.yaml} + bar: + get_file: + 'http://localhost/bar.yaml' + resource2: + type: type1 + properties: + baz: + - {get_file: baz/baz1.yaml} + - {get_file: baz/baz2.yaml} + - {get_file: baz/baz3.yaml} + ignored_list: {get_file: [ignore, me]} + ignored_dict: {get_file: {ignore: me}} + ignored_none: {get_file: } + ''' + + def setUp(self): + super(TestTemplateGetFileFunctions, self).setUp() + self.m = mox.Mox() + + self.addCleanup(self.m.VerifyAll) + self.addCleanup(self.m.UnsetStubs) + + def test_hot_template(self): + self.m.StubOutWithMock(urlutils, 'urlopen') + + tmpl_file = '/home/my/dir/template.yaml' + url = 'file:///home/my/dir/template.yaml' + urlutils.urlopen(url).AndReturn( + six.StringIO(self.hot_template)) + urlutils.urlopen( + 'http://localhost/bar.yaml').InAnyOrder().AndReturn( + six.StringIO('bar contents')) + urlutils.urlopen( + 'file:///home/my/dir/foo.yaml').InAnyOrder().AndReturn( + six.StringIO('foo contents')) + urlutils.urlopen( + 'file:///home/my/dir/baz/baz1.yaml').InAnyOrder().AndReturn( + six.StringIO('baz1 contents')) + urlutils.urlopen( + 'file:///home/my/dir/baz/baz2.yaml').InAnyOrder().AndReturn( + six.StringIO('baz2 contents')) + urlutils.urlopen( + 'file:///home/my/dir/baz/baz3.yaml').InAnyOrder().AndReturn( + six.StringIO('baz3 contents')) + + self.m.ReplayAll() + + files, tmpl_parsed = template_utils.get_template_contents( + template_file=tmpl_file) + + self.assertEqual({ + 'http://localhost/bar.yaml': 'bar contents', + 'file:///home/my/dir/foo.yaml': 'foo contents', + 'file:///home/my/dir/baz/baz1.yaml': 'baz1 contents', + 'file:///home/my/dir/baz/baz2.yaml': 'baz2 contents', + 'file:///home/my/dir/baz/baz3.yaml': 'baz3 contents', + }, files) + self.assertEqual({ + 'heat_template_version': '2013-05-23', + 'resources': { + 'resource1': { + 'type': 'type1', + 'properties': { + 'bar': {'get_file': 'http://localhost/bar.yaml'}, + 'foo': {'get_file': 'file:///home/my/dir/foo.yaml'}, + }, + }, + 'resource2': { + 'type': 'type1', + 'properties': { + 'baz': [ + {'get_file': 'file:///home/my/dir/baz/baz1.yaml'}, + {'get_file': 'file:///home/my/dir/baz/baz2.yaml'}, + {'get_file': 'file:///home/my/dir/baz/baz3.yaml'}, + ], + 'ignored_list': {'get_file': ['ignore', 'me']}, + 'ignored_dict': {'get_file': {'ignore': 'me'}}, + 'ignored_none': {'get_file': None}, + }, + } + } + }, tmpl_parsed) + + self.m.VerifyAll() + + class TestURLFunctions(testtools.TestCase): def setUp(self): diff --git a/heatclient/v1/shell.py b/heatclient/v1/shell.py index 91f1a97e..11b22fa6 100644 --- a/heatclient/v1/shell.py +++ b/heatclient/v1/shell.py @@ -24,7 +24,7 @@ import heatclient.exc as exc @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -49,7 +49,7 @@ def do_create(hc, args): @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -69,7 +69,12 @@ def do_create(hc, args): help='Name of the stack to create.') def do_stack_create(hc, args): '''Create the stack.''' - files, env = template_utils.process_environment_and_files( + tpl_files, template = template_utils.get_template_contents( + args.template_file, + args.template_url, + args.template_object, + hc.http_client.raw_request) + env_files, env = template_utils.process_environment_and_files( env_path=args.environment_file) fields = { @@ -77,12 +82,8 @@ def do_stack_create(hc, args): 'timeout_mins': args.create_timeout, 'disable_rollback': not(args.enable_rollback), 'parameters': utils.format_parameters(args.parameters), - 'template': template_utils.get_template_contents( - args.template_file, - args.template_url, - args.template_object, - hc.http_client.raw_request), - 'files': files, + 'template': template, + 'files': dict(tpl_files.items() + env_files.items()), 'environment': env } @@ -171,7 +172,7 @@ def do_stack_show(hc, args): @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -191,7 +192,7 @@ def do_update(hc, args): @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -206,18 +207,21 @@ def do_update(hc, args): help='Name or ID of stack to update.') def do_stack_update(hc, args): '''Update the stack.''' - files, env = template_utils.process_environment_and_files( + + tpl_files, template = template_utils.get_template_contents( + args.template_file, + args.template_url, + args.template_object, + hc.http_client.raw_request) + + env_files, env = template_utils.process_environment_and_files( env_path=args.environment_file) fields = { 'stack_id': args.id, 'parameters': utils.format_parameters(args.parameters), - 'template': template_utils.get_template_contents( - args.template_file, - args.template_url, - args.template_object, - hc.http_client.raw_request), - 'files': files, + 'template': template, + 'files': dict(tpl_files.items() + env_files.items()), 'environment': env } @@ -285,7 +289,7 @@ def do_template_show(hc, args): help='URL of template.') @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-o', '--template-object', metavar='', help='URL to retrieve template object (e.g from swift)') @@ -303,7 +307,7 @@ def do_validate(hc, args): help='URL of template.') @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-o', '--template-object', metavar='', help='URL to retrieve template object (e.g from swift)') @@ -314,16 +318,19 @@ def do_validate(hc, args): action='append') def do_template_validate(hc, args): '''Validate a template with parameters.''' - files, env = template_utils.process_environment_and_files( + + tpl_files, template = template_utils.get_template_contents( + args.template_file, + args.template_url, + args.template_object, + hc.http_client.raw_request) + + env_files, env = template_utils.process_environment_and_files( env_path=args.environment_file) fields = { 'parameters': utils.format_parameters(args.parameters), - 'template': template_utils.get_template_contents( - args.template_file, - args.template_url, - args.template_object, - hc.http_client.raw_request), - 'files': files, + 'template': template, + 'files': dict(tpl_files.items() + env_files.items()), 'environment': env }