From e86a40e3ba1510cb196e3a080757d78c5c80c701 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Sun, 12 Jan 2014 12:57:22 +1300 Subject: [PATCH] Populate files with content from get_file function calls This is the client-side implementation of the get_file HOT intrinsic function. The data for each resource is recursively crawled to look for {get_file: } calls. The content for the URL specified by get_file is fetched and the content stored in the files field in the API request. If a relative path is specified, that path is converted to an absolute URL, and that URL is used as the key for the files content entry. The template is also modified to replace the relative path with the absolute URL. This implements the client-side portion of blueprint get-file Change-Id: If1da98578bb2919e3c8325442224376d3fadb0bc --- heatclient/common/template_utils.py | 86 ++++++++++++++------ heatclient/tests/test_template_utils.py | 104 +++++++++++++++++++++++- heatclient/v1/shell.py | 61 ++++++++------ 3 files changed, 194 insertions(+), 57 deletions(-) diff --git a/heatclient/common/template_utils.py b/heatclient/common/template_utils.py index bd54c9de..cdc71ccd 100644 --- a/heatclient/common/template_utils.py +++ b/heatclient/common/template_utils.py @@ -14,6 +14,7 @@ # under the License. import os +import six import urllib from heatclient.common import environment_format @@ -46,28 +47,60 @@ def get_template_contents(template_file=None, template_url=None, % template_url) try: - return template_format.parse(tpl) + template = template_format.parse(tpl) except ValueError as e: raise exc.CommandError( 'Error parsing template %s %s' % (template_url, e)) + files = {} + tmpl_base_url = base_url_for_url(template_url) + resolve_template_get_files(template, files, tmpl_base_url) + return files, template -def get_file_contents(from_dict, files, base_url=None, - ignore_if=None): - for key, value in iter(from_dict.items()): - if ignore_if and ignore_if(key, value): - continue - if base_url and not base_url.endswith('/'): - base_url = base_url + '/' +def resolve_template_get_files(template, files, template_base_url): - str_url = urlutils.urljoin(base_url, value) - try: - files[str_url] = urlutils.urlopen(str_url).read() - except urlutils.URLError: - raise exc.CommandError('Could not fetch %s from the environment' - % str_url) - from_dict[key] = str_url + def ignore_if(key, value): + if key != 'get_file': + return True + if not isinstance(value, six.string_types): + return True + + def recurse_if(value): + return isinstance(value, (dict, list)) + + get_file_contents(template.get('resources'), files, template_base_url, + ignore_if, recurse_if) + + +def get_file_contents(from_data, files, base_url=None, + ignore_if=None, recurse_if=None): + + if recurse_if and recurse_if(from_data): + if isinstance(from_data, dict): + recurse_data = from_data.itervalues() + else: + recurse_data = from_data + for value in recurse_data: + get_file_contents(value, files, base_url, ignore_if, recurse_if) + + if isinstance(from_data, dict): + for key, value in iter(from_data.items()): + if ignore_if and ignore_if(key, value): + continue + + if base_url and not base_url.endswith('/'): + base_url = base_url + '/' + + str_url = urlutils.urljoin(base_url, value) + try: + files[str_url] = urlutils.urlopen(str_url).read() + except urlutils.URLError: + raise exc.CommandError('Could not fetch contents for %s' + % str_url) + + # replace the data value with the normalised absolute URL + from_data[key] = str_url def base_url_for_url(url): @@ -83,22 +116,21 @@ def normalise_file_path_to_url(path): return urlutils.urljoin('file:', urllib.pathname2url(path)) -def process_environment_and_files(env_path=None, template_path=None): +def process_environment_and_files(env_path=None, template=None, + template_url=None): files = {} env = {} - if not env_path: - return files, env + if env_path: + env_url = normalise_file_path_to_url(env_path) + env_base_url = base_url_for_url(env_url) + raw_env = urlutils.urlopen(env_url).read() + env = environment_format.parse(raw_env) - env_url = normalise_file_path_to_url(env_path) - env_base_url = base_url_for_url(env_url) - raw_env = urlutils.urlopen(env_url).read() - env = environment_format.parse(raw_env) - - resolve_environment_urls( - env.get('resource_registry'), - files, - env_base_url) + resolve_environment_urls( + env.get('resource_registry'), + files, + env_base_url) return files, env diff --git a/heatclient/tests/test_template_utils.py b/heatclient/tests/test_template_utils.py index 2b9b2ae3..2b2fce1e 100644 --- a/heatclient/tests/test_template_utils.py +++ b/heatclient/tests/test_template_utils.py @@ -269,9 +269,10 @@ class TestGetTemplateContents(testtools.TestCase): tmpl_file.write(tmpl) tmpl_file.flush() - tmpl_parsed = template_utils.get_template_contents( + files, tmpl_parsed = template_utils.get_template_contents( tmpl_file.name) self.assertEqual({"foo": "bar"}, tmpl_parsed) + self.assertEqual({}, files) def test_get_template_contents_file_empty(self): with tempfile.NamedTemporaryFile() as tmpl_file: @@ -316,8 +317,10 @@ class TestGetTemplateContents(testtools.TestCase): urlutils.urlopen(url).AndReturn(six.StringIO(tmpl)) self.m.ReplayAll() - tmpl_parsed = template_utils.get_template_contents(template_url=url) + files, tmpl_parsed = template_utils.get_template_contents( + template_url=url) self.assertEqual({"foo": "bar"}, tmpl_parsed) + self.assertEqual({}, files) def test_get_template_contents_object(self): tmpl = '{"foo": "bar"}' @@ -332,14 +335,109 @@ class TestGetTemplateContents(testtools.TestCase): self.assertEqual('http://no.where/path/to/a.yaml', object_url) return tmpl - tmpl_parsed = template_utils.get_template_contents( + files, tmpl_parsed = template_utils.get_template_contents( template_object=url, object_request=object_request) self.assertEqual({"foo": "bar"}, tmpl_parsed) + self.assertEqual({}, files) self.assertTrue(self.object_requested) +class TestTemplateGetFileFunctions(testtools.TestCase): + + hot_template = '''heat_template_version: 2013-05-23 +resources: + resource1: + type: type1 + properties: + foo: {get_file: foo.yaml} + bar: + get_file: + 'http://localhost/bar.yaml' + resource2: + type: type1 + properties: + baz: + - {get_file: baz/baz1.yaml} + - {get_file: baz/baz2.yaml} + - {get_file: baz/baz3.yaml} + ignored_list: {get_file: [ignore, me]} + ignored_dict: {get_file: {ignore: me}} + ignored_none: {get_file: } + ''' + + def setUp(self): + super(TestTemplateGetFileFunctions, self).setUp() + self.m = mox.Mox() + + self.addCleanup(self.m.VerifyAll) + self.addCleanup(self.m.UnsetStubs) + + def test_hot_template(self): + self.m.StubOutWithMock(urlutils, 'urlopen') + + tmpl_file = '/home/my/dir/template.yaml' + url = 'file:///home/my/dir/template.yaml' + urlutils.urlopen(url).AndReturn( + six.StringIO(self.hot_template)) + urlutils.urlopen( + 'http://localhost/bar.yaml').InAnyOrder().AndReturn( + six.StringIO('bar contents')) + urlutils.urlopen( + 'file:///home/my/dir/foo.yaml').InAnyOrder().AndReturn( + six.StringIO('foo contents')) + urlutils.urlopen( + 'file:///home/my/dir/baz/baz1.yaml').InAnyOrder().AndReturn( + six.StringIO('baz1 contents')) + urlutils.urlopen( + 'file:///home/my/dir/baz/baz2.yaml').InAnyOrder().AndReturn( + six.StringIO('baz2 contents')) + urlutils.urlopen( + 'file:///home/my/dir/baz/baz3.yaml').InAnyOrder().AndReturn( + six.StringIO('baz3 contents')) + + self.m.ReplayAll() + + files, tmpl_parsed = template_utils.get_template_contents( + template_file=tmpl_file) + + self.assertEqual({ + 'http://localhost/bar.yaml': 'bar contents', + 'file:///home/my/dir/foo.yaml': 'foo contents', + 'file:///home/my/dir/baz/baz1.yaml': 'baz1 contents', + 'file:///home/my/dir/baz/baz2.yaml': 'baz2 contents', + 'file:///home/my/dir/baz/baz3.yaml': 'baz3 contents', + }, files) + self.assertEqual({ + 'heat_template_version': '2013-05-23', + 'resources': { + 'resource1': { + 'type': 'type1', + 'properties': { + 'bar': {'get_file': 'http://localhost/bar.yaml'}, + 'foo': {'get_file': 'file:///home/my/dir/foo.yaml'}, + }, + }, + 'resource2': { + 'type': 'type1', + 'properties': { + 'baz': [ + {'get_file': 'file:///home/my/dir/baz/baz1.yaml'}, + {'get_file': 'file:///home/my/dir/baz/baz2.yaml'}, + {'get_file': 'file:///home/my/dir/baz/baz3.yaml'}, + ], + 'ignored_list': {'get_file': ['ignore', 'me']}, + 'ignored_dict': {'get_file': {'ignore': 'me'}}, + 'ignored_none': {'get_file': None}, + }, + } + } + }, tmpl_parsed) + + self.m.VerifyAll() + + class TestURLFunctions(testtools.TestCase): def setUp(self): diff --git a/heatclient/v1/shell.py b/heatclient/v1/shell.py index 91f1a97e..11b22fa6 100644 --- a/heatclient/v1/shell.py +++ b/heatclient/v1/shell.py @@ -24,7 +24,7 @@ import heatclient.exc as exc @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -49,7 +49,7 @@ def do_create(hc, args): @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -69,7 +69,12 @@ def do_create(hc, args): help='Name of the stack to create.') def do_stack_create(hc, args): '''Create the stack.''' - files, env = template_utils.process_environment_and_files( + tpl_files, template = template_utils.get_template_contents( + args.template_file, + args.template_url, + args.template_object, + hc.http_client.raw_request) + env_files, env = template_utils.process_environment_and_files( env_path=args.environment_file) fields = { @@ -77,12 +82,8 @@ def do_stack_create(hc, args): 'timeout_mins': args.create_timeout, 'disable_rollback': not(args.enable_rollback), 'parameters': utils.format_parameters(args.parameters), - 'template': template_utils.get_template_contents( - args.template_file, - args.template_url, - args.template_object, - hc.http_client.raw_request), - 'files': files, + 'template': template, + 'files': dict(tpl_files.items() + env_files.items()), 'environment': env } @@ -171,7 +172,7 @@ def do_stack_show(hc, args): @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -191,7 +192,7 @@ def do_update(hc, args): @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-u', '--template-url', metavar='', help='URL of template.') @@ -206,18 +207,21 @@ def do_update(hc, args): help='Name or ID of stack to update.') def do_stack_update(hc, args): '''Update the stack.''' - files, env = template_utils.process_environment_and_files( + + tpl_files, template = template_utils.get_template_contents( + args.template_file, + args.template_url, + args.template_object, + hc.http_client.raw_request) + + env_files, env = template_utils.process_environment_and_files( env_path=args.environment_file) fields = { 'stack_id': args.id, 'parameters': utils.format_parameters(args.parameters), - 'template': template_utils.get_template_contents( - args.template_file, - args.template_url, - args.template_object, - hc.http_client.raw_request), - 'files': files, + 'template': template, + 'files': dict(tpl_files.items() + env_files.items()), 'environment': env } @@ -285,7 +289,7 @@ def do_template_show(hc, args): help='URL of template.') @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-o', '--template-object', metavar='', help='URL to retrieve template object (e.g from swift)') @@ -303,7 +307,7 @@ def do_validate(hc, args): help='URL of template.') @utils.arg('-f', '--template-file', metavar='', help='Path to the template.') -@utils.arg('-e', '--environment-file', metavar='', +@utils.arg('-e', '--environment-file', metavar='', help='Path to the environment.') @utils.arg('-o', '--template-object', metavar='', help='URL to retrieve template object (e.g from swift)') @@ -314,16 +318,19 @@ def do_validate(hc, args): action='append') def do_template_validate(hc, args): '''Validate a template with parameters.''' - files, env = template_utils.process_environment_and_files( + + tpl_files, template = template_utils.get_template_contents( + args.template_file, + args.template_url, + args.template_object, + hc.http_client.raw_request) + + env_files, env = template_utils.process_environment_and_files( env_path=args.environment_file) fields = { 'parameters': utils.format_parameters(args.parameters), - 'template': template_utils.get_template_contents( - args.template_file, - args.template_url, - args.template_object, - hc.http_client.raw_request), - 'files': files, + 'template': template, + 'files': dict(tpl_files.items() + env_files.items()), 'environment': env }