Add files-container option for stack create and update

If files-container option is specified:

- All template/env files would be fetched by the heat engine
  relative to the files_container and no local files other
  than the root template would be sent to server.
- Relative path of environment files would be sent in the
  environment_files list.

Also adds the option to template validate.

Change-Id: I1a703ab8798a003365be650886bb78be5af472b7
Story: #1755453
Task: 19319

heatclient/common/template_utils.py

@@ -27,7 +27,8 @@ from heatclient.common import utils
 from heatclient import exc
 
 
-def process_template_path(template_path, object_request=None, existing=False):
+def process_template_path(template_path, object_request=None,
+                          existing=False, fetch_child=True):
     """Read template from template path.
 
     Attempt to read template first as a file or url. If that is unsuccessful,
@@ -37,17 +38,20 @@ def process_template_path(template_path, object_request=None, existing=False):
     :param object_request: custom object request function used to get template
                            if local or uri path fails
     :param existing: if the current stack's template should be used
+    :param fetch_child: Whether to fetch the child templates
     :returns: get_file dict and template contents
     :raises: error.URLError
     """
     try:
         return get_template_contents(template_file=template_path,
-                                     existing=existing)
+                                     existing=existing,
+                                     fetch_child=fetch_child)
     except error.URLError as template_file_exc:
         try:
             return get_template_contents(template_object=template_path,
                                          object_request=object_request,
-                                         existing=existing)
+                                         existing=existing,
+                                         fetch_child=fetch_child)
         except exc.HTTPNotFound:
             # The initial exception gives the user better failure context.
             raise template_file_exc
@@ -55,7 +59,8 @@ def process_template_path(template_path, object_request=None, existing=False):
 
 def get_template_contents(template_file=None, template_url=None,
                           template_object=None, object_request=None,
-                          files=None, existing=False):
+                          files=None, existing=False,
+                          fetch_child=True):
 
     is_object = False
     # Transform a bare file path to a file:// URL.
@@ -93,12 +98,13 @@ def get_template_contents(template_file=None, template_url=None,
     except ValueError as e:
         raise exc.CommandError(_('Error parsing template %(url)s %(error)s') %
                                {'url': template_url, 'error': e})
-
-    tmpl_base_url = utils.base_url_for_url(template_url)
     if files is None:
         files = {}
-    resolve_template_get_files(template, files, tmpl_base_url, is_object,
+
-                               object_request)
+    if fetch_child:
+        tmpl_base_url = utils.base_url_for_url(template_url)
+        resolve_template_get_files(template, files, tmpl_base_url, is_object,
+                                   object_request)
     return files, template
 
 
@@ -212,7 +218,8 @@ def process_multiple_environments_and_files(env_paths=None, template=None,
                                             template_url=None,
                                             env_path_is_object=None,
                                             object_request=None,
-                                            env_list_tracker=None):
+                                            env_list_tracker=None,
+                                            fetch_env_files=True):
     """Reads one or more environment files.
 
     Reads in each specified environment file and returns a dictionary
@@ -239,6 +246,7 @@ def process_multiple_environments_and_files(env_paths=None, template=None,
     :type  env_list_tracker: list or None
     :return: tuple of files dict and a dict of the consolidated environment
     :rtype:  tuple
+    :param fetch_env_files: fetch env_files or leave it to server
     """
     merged_files = {}
     merged_env = {}
@@ -249,24 +257,28 @@ def process_multiple_environments_and_files(env_paths=None, template=None,
 
     if env_paths:
         for env_path in env_paths:
-            files, env = process_environment_and_files(
+            if fetch_env_files:
-                env_path=env_path,
+                files, env = process_environment_and_files(
-                template=template,
+                    env_path=env_path,
-                template_url=template_url,
+                    template=template,
-                env_path_is_object=env_path_is_object,
+                    template_url=template_url,
-                object_request=object_request,
+                    env_path_is_object=env_path_is_object,
-                include_env_in_files=include_env_in_files)
+                    object_request=object_request,
+                    include_env_in_files=include_env_in_files)
 
-            # 'files' looks like {"filename1": contents, "filename2": contents}
+                # 'files' looks like:
-            # so a simple update is enough for merging
+                # {"filename1": contents, "filename2": contents}
-            merged_files.update(files)
+                # so a simple update is enough for merging
+                merged_files.update(files)
 
-            # 'env' can be a deeply nested dictionary, so a simple update is
+                # 'env' can be a deeply nested dictionary, so a simple
-            # not enough
+                # update is not enough
-            merged_env = deep_update(merged_env, env)
+                merged_env = deep_update(merged_env, env)
+                env_url = utils.normalise_file_path_to_url(env_path)
+            else:
+                env_url = env_path
 
             if env_list_tracker is not None:
-                env_url = utils.normalise_file_path_to_url(env_path)
                 env_list_tracker.append(env_url)
 
     return merged_files, merged_env

heatclient/osc/v1/stack.py

@@ -46,6 +46,13 @@ class CreateStack(command.ShowOne):
             action='append',
             help=_('Path to the environment. Can be specified multiple times')
         )
+        parser.add_argument(
+            '-s', '--files-container',
+            metavar='<files-container>',
+            help=_('Swift files container name. Local files other than '
+                   'root template would be ignored. If other files are not '
+                   'found in swift, heat engine would raise an error.')
+        )
         parser.add_argument(
             '--timeout',
             metavar='<timeout>',
@@ -121,13 +128,15 @@ class CreateStack(command.ShowOne):
 
         tpl_files, template = template_utils.process_template_path(
             parsed_args.template,
-            object_request=http.authenticated_fetcher(client))
+            object_request=http.authenticated_fetcher(client),
+            fetch_child=parsed_args.files_container is None)
 
         env_files_list = []
         env_files, env = (
             template_utils.process_multiple_environments_and_files(
                 env_paths=parsed_args.environment,
-                env_list_tracker=env_files_list))
+                env_list_tracker=env_files_list,
+                fetch_env_files=parsed_args.files_container is None))
 
         parameters = heat_utils.format_all_parameters(
             parsed_args.parameter,
@@ -151,6 +160,9 @@ class CreateStack(command.ShowOne):
         if env_files_list:
             fields['environment_files'] = env_files_list
 
+        if parsed_args.files_container:
+            fields['files_container'] = parsed_args.files_container
+
         if parsed_args.tags:
             fields['tags'] = parsed_args.tags
         if parsed_args.timeout:
@@ -201,6 +213,13 @@ class UpdateStack(command.ShowOne):
             '-t', '--template', metavar='<template>',
             help=_('Path to the template')
         )
+        parser.add_argument(
+            '-s', '--files-container',
+            metavar='<files-container>',
+            help=_('Swift files container name. Local files other than '
+                   'root template would be ignored. If other files are not '
+                   'found in swift, heat engine would raise an error.')
+        )
         parser.add_argument(
             '-e', '--environment', metavar='<environment>',
             action='append',
@@ -298,13 +317,15 @@ class UpdateStack(command.ShowOne):
         tpl_files, template = template_utils.process_template_path(
             parsed_args.template,
             object_request=http.authenticated_fetcher(client),
-            existing=parsed_args.existing)
+            existing=parsed_args.existing,
+            fetch_child=parsed_args.files_container is None)
 
         env_files_list = []
         env_files, env = (
             template_utils.process_multiple_environments_and_files(
                 env_paths=parsed_args.environment,
-                env_list_tracker=env_files_list))
+                env_list_tracker=env_files_list,
+                fetch_env_files=parsed_args.files_container is None))
 
         parameters = heat_utils.format_all_parameters(
             parsed_args.parameter,
@@ -328,6 +349,9 @@ class UpdateStack(command.ShowOne):
         if env_files_list:
             fields['environment_files'] = env_files_list
 
+        if parsed_args.files_container:
+            fields['files_container'] = parsed_args.files_container
+
         if parsed_args.tags:
             fields['tags'] = parsed_args.tags
         if parsed_args.timeout:

heatclient/osc/v1/template.py

@@ -122,6 +122,13 @@ class Validate(format_utils.YamlFormat):
             help=_('Parameter values used to create the stack. This can be '
                    'specified multiple times')
         )
+        parser.add_argument(
+            '-s', '--files-container',
+            metavar='<files-container>',
+            help=_('Swift files container name. Local files other than '
+                   'root template would be ignored. If other files are not '
+                   'found in swift, heat engine would raise an error.')
+        )
         parser.add_argument(
             '--ignore-errors',
             metavar='<error1,error2,...>',
@@ -145,11 +152,13 @@ class Validate(format_utils.YamlFormat):
 def _validate(heat_client, args):
     tpl_files, template = template_utils.process_template_path(
         args.template,
-        object_request=http.authenticated_fetcher(heat_client))
+        object_request=http.authenticated_fetcher(heat_client),
+        fetch_child=args.files_container is None)
 
     env_files_list = []
     env_files, env = template_utils.process_multiple_environments_and_files(
-        env_paths=args.environment, env_list_tracker=env_files_list)
+        env_paths=args.environment, env_list_tracker=env_files_list,
+        fetch_env_files=args.files_container is None)
 
     fields = {
         'template': template,
@@ -168,6 +177,9 @@ def _validate(heat_client, args):
     if args.show_nested:
         fields['show_nested'] = args.show_nested
 
+    if args.files_container:
+        fields['files_container'] = args.files_container
+
     validation = heat_client.stacks.validate(**fields)
     data = list(six.itervalues(validation))
     columns = list(six.iterkeys(validation))

heatclient/tests/unit/test_template_utils.py

@@ -127,6 +127,18 @@ class ShellEnvironmentTest(testtools.TestCase):
             mock.call('file:///home/my/dir/a.yaml')
         ])
 
+    def test_process_multiple_environment_files_container(self):
+
+        env_list_tracker = []
+        env_paths = ['/home/my/dir/env.yaml']
+        files, env = template_utils.process_multiple_environments_and_files(
+            env_paths, env_list_tracker=env_list_tracker,
+            fetch_env_files=False)
+
+        self.assertEqual(env_paths, env_list_tracker)
+        self.assertEqual({}, files)
+        self.assertEqual({}, env)
+
     @mock.patch('six.moves.urllib.request.urlopen')
     def test_process_environment_relative_file_up(self, mock_url):
 

releasenotes/notes/add-files-container-stack-create-update-324b931f8f474b3d.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Adds ``files-container`` option for stack create, update and
+    template validate with openstackclient. If specified, no local
+    files other than root template would be sent to heat engine.
+    Heat engine would try and download the all other files relative
+    to the ``files-container``, else raise an error.