Add handle get_file when launch stack from horizon

when get_file is contained in template, the stack create/update/preview
will fail due to No content found in the "files" section.
So added handle get_file code.

Change-Id: I6f125f9e5f3f53f630ab0d4f3f00631e6850e905
Closes-Bug: #1512564

horizon/exceptions.py

@@ -170,6 +170,26 @@ class AlreadyExists(HorizonException):
         return self.msg % self.attrs
 
 
+@six.python_2_unicode_compatible
+class GetFileError(HorizonException):
+    """Exception to be raised when the value of get_file did not start with
+    https:// or http://
+    """
+    def __init__(self, name, resource_type):
+        self.attrs = {"name": name, "resource": resource_type}
+        self.msg = _('The value of %(resource)s is %(name)s inside the '
+                     'template. When launching a stack from this interface,'
+                     ' the value must start with "http://" or "https://"')
+
+    def __repr__(self):
+        return '<%s name=%r resource_type=%r>' % (self.__class__.__name__,
+                                                  self.attrs['name'],
+                                                  self.attrs['resource_type'])
+
+    def __str__(self):
+        return self.msg % self.attrs
+
+
 class ConfigurationError(HorizonException):
     """Exception to be raised when invalid settings have been provided."""
     pass
@@ -203,6 +223,7 @@ class HandledException(HorizonException):
 UNAUTHORIZED = tuple(HORIZON_CONFIG['exceptions']['unauthorized'])
 UNAUTHORIZED += (NotAuthorized,)
 NOT_FOUND = tuple(HORIZON_CONFIG['exceptions']['not_found'])
+NOT_FOUND += (GetFileError,)
 RECOVERABLE = (AlreadyExists, Conflict, NotAvailable, ServiceCatalogException)
 RECOVERABLE += tuple(HORIZON_CONFIG['exceptions']['recoverable'])
 

openstack_dashboard/api/heat.py

@@ -10,9 +10,18 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-from django.conf import settings
+import contextlib
-from heatclient import client as heat_client
+import six
+from six.moves.urllib import request
 
+from django.conf import settings
+from oslo_serialization import jsonutils
+
+from heatclient import client as heat_client
+from heatclient.common import template_format
+from heatclient.common import template_utils
+from heatclient.common import utils as heat_utils
+from horizon import exceptions
 from horizon.utils import functions as utils
 from horizon.utils.memoized import memoized  # noqa
 from openstack_dashboard.api import base
@@ -82,6 +91,57 @@ def stacks_list(request, marker=None, sort_dir='desc', sort_key='created_at',
     return (stacks, has_more_data, has_prev_data)
 
 
+def _ignore_if(key, value):
+    if key != 'get_file' and key != 'type':
+        return True
+    if not isinstance(value, six.string_types):
+        return True
+    if (key == 'type' and
+            not value.endswith(('.yaml', '.template'))):
+        return True
+    return False
+
+
+def get_template_files(template_data=None, template_url=None):
+    if template_data:
+        tpl = template_data
+    elif template_url:
+        with contextlib.closing(request.urlopen(template_url)) as u:
+            tpl = u.read()
+    else:
+        return {}, None
+    if not tpl:
+        return {}, None
+    if isinstance(tpl, six.binary_type):
+        tpl = tpl.decode('utf-8')
+    template = template_format.parse(tpl)
+    files = {}
+    _get_file_contents(template, files)
+    return files, template
+
+
+def _get_file_contents(from_data, files):
+    if not isinstance(from_data, (dict, list)):
+        return
+    if isinstance(from_data, dict):
+        recurse_data = six.itervalues(from_data)
+        for key, value in six.iteritems(from_data):
+            if _ignore_if(key, value):
+                continue
+            if not value.startswith(('http://', 'https://')):
+                raise exceptions.GetFileError(value, 'get_file')
+            if value not in files:
+                file_content = heat_utils.read_url_content(value)
+                if template_utils.is_template(file_content):
+                    template = get_template_files(template_url=value)[1]
+                    file_content = jsonutils.dumps(template)
+                files[value] = file_content
+    else:
+        recurse_data = from_data
+    for value in recurse_data:
+        _get_file_contents(value, files)
+
+
 def stack_delete(request, stack_id):
     return heatclient(request).stacks.delete(stack_id)
 

openstack_dashboard/dashboards/project/stacks/forms.py

@@ -139,17 +139,18 @@ class TemplateForm(forms.SelfHandlingForm):
 
         # Validate the template and get back the params.
         kwargs = {}
-        if cleaned['template_data']:
-            kwargs['template'] = cleaned['template_data']
-        else:
-            kwargs['template_url'] = cleaned['template_url']
-
         if cleaned['environment_data']:
             kwargs['environment'] = cleaned['environment_data']
-
         try:
+            files, tpl =\
+                api.heat.get_template_files(cleaned.get('template_data'),
+                                            cleaned.get('template_url'))
+            kwargs['files'] = files
+            kwargs['template'] = tpl
             validated = api.heat.template_validate(self.request, **kwargs)
             cleaned['template_validate'] = validated
+            cleaned['template_validate']['files'] = files
+            cleaned['template_validate']['template'] = tpl
         except Exception as e:
             raise forms.ValidationError(six.text_type(e))
 
@@ -209,9 +210,7 @@ class TemplateForm(forms.SelfHandlingForm):
 
     def create_kwargs(self, data):
         kwargs = {'parameters': data['template_validate'],
-                  'environment_data': data['environment_data'],
+                  'environment_data': data['environment_data']}
-                  'template_data': data['template_data'],
-                  'template_url': data['template_url']}
         if data.get('stack_id'):
             kwargs['stack_id'] = data['stack_id']
         return kwargs
@@ -250,12 +249,6 @@ class CreateStackForm(forms.SelfHandlingForm):
     class Meta(object):
         name = _('Create Stack')
 
-    template_data = forms.CharField(
-        widget=forms.widgets.HiddenInput,
-        required=False)
-    template_url = forms.CharField(
-        widget=forms.widgets.HiddenInput,
-        required=False)
     environment_data = forms.CharField(
         widget=forms.widgets.HiddenInput,
         required=False)
@@ -375,15 +368,12 @@ class CreateStackForm(forms.SelfHandlingForm):
             'timeout_mins': data.get('timeout_mins'),
             'disable_rollback': not(data.get('enable_rollback')),
             'parameters': dict(params_list),
+            'files': json.loads(data.get('parameters')).get('files'),
+            'template': json.loads(data.get('parameters')).get('template')
         }
         if data.get('password'):
             fields['password'] = data.get('password')
 
-        if data.get('template_data'):
-            fields['template'] = data.get('template_data')
-        else:
-            fields['template_url'] = data.get('template_url')
-
         if data.get('environment_data'):
             fields['environment'] = data.get('environment_data')
 
@@ -430,19 +420,12 @@ class EditStackForm(CreateStackForm):
             'timeout_mins': data.get('timeout_mins'),
             'disable_rollback': not(data.get('enable_rollback')),
             'parameters': dict(params_list),
+            'files': json.loads(data.get('parameters')).get('files'),
+            'template': json.loads(data.get('parameters')).get('template')
         }
         if data.get('password'):
             fields['password'] = data.get('password')
 
-        # if the user went directly to this form, resubmit the existing
-        # template data. otherwise, submit what they had from the first form
-        if data.get('template_data'):
-            fields['template'] = data.get('template_data')
-        elif data.get('template_url'):
-            fields['template_url'] = data.get('template_url')
-        elif data.get('parameters'):
-            fields['template'] = data.get('parameters')
-
         try:
             api.heat.stack_update(self.request, stack_id=stack_id, **fields)
             messages.success(request, _("Stack update started."))
@@ -469,13 +452,10 @@ class PreviewStackForm(CreateStackForm):
             'timeout_mins': data.get('timeout_mins'),
             'disable_rollback': not(data.get('enable_rollback')),
             'parameters': dict(params_list),
+            'files': json.loads(data.get('parameters')).get('files'),
+            'template': json.loads(data.get('parameters')).get('template')
         }
 
-        if data.get('template_data'):
-            fields['template'] = data.get('template_data')
-        else:
-            fields['template_url'] = data.get('template_url')
-
         if data.get('environment_data'):
             fields['environment'] = data.get('environment_data')
 

openstack_dashboard/dashboards/project/stacks/tests.py

@@ -24,6 +24,7 @@ from django.utils import html
 from mox3.mox import IsA  # noqa
 import six
 
+from heatclient.common import template_format as hc_format
 from openstack_dashboard import api
 from openstack_dashboard.test import helpers as test
 
@@ -230,16 +231,18 @@ class StackTests(test.TestCase):
         stack = self.stacks.first()
 
         api.heat.template_validate(IsA(http.HttpRequest),
-                                   template=template.data) \
+                                   files={},
+                                   template=hc_format.parse(template.data)) \
            .AndReturn(json.loads(template.validate))
 
         api.heat.stack_create(IsA(http.HttpRequest),
                               stack_name=stack.stack_name,
                               timeout_mins=60,
                               disable_rollback=True,
-                              template=template.data,
+                              template=None,
                               parameters=IsA(dict),
-                              password='password')
+                              password='password',
+                              files=None)
         api.neutron.network_list_for_tenant(IsA(http.HttpRequest),
                                             self.tenant.id) \
             .AndReturn(self.networks.list())
@@ -287,7 +290,8 @@ class StackTests(test.TestCase):
         stack = self.stacks.first()
 
         api.heat.template_validate(IsA(http.HttpRequest),
-                                   template=template.data,
+                                   files={},
+                                   template=hc_format.parse(template.data),
                                    environment=environment.data) \
            .AndReturn(json.loads(template.validate))
 
@@ -295,10 +299,11 @@ class StackTests(test.TestCase):
                               stack_name=stack.stack_name,
                               timeout_mins=60,
                               disable_rollback=True,
-                              template=template.data,
+                              template=None,
                               environment=environment.data,
                               parameters=IsA(dict),
-                              password='password')
+                              password='password',
+                              files=None)
         api.neutron.network_list_for_tenant(IsA(http.HttpRequest),
                                             self.tenant.id) \
             .AndReturn(self.networks.list())
@@ -371,7 +376,8 @@ class StackTests(test.TestCase):
             }
         }
         api.heat.template_validate(IsA(http.HttpRequest),
-                                   template=template['data']) \
+                                   files={},
+                                   template=hc_format.parse(template['data'])) \
            .AndReturn(template['validate'])
 
         self.mox.ReplayAll()
@@ -448,7 +454,8 @@ class StackTests(test.TestCase):
             }
         }
         api.heat.template_validate(IsA(http.HttpRequest),
-                                   template=template['data']) \
+                                   files={},
+                                   template=hc_format.parse(template['data'])) \
            .AndReturn(template['validate'])
 
         self.mox.ReplayAll()
@@ -522,20 +529,22 @@ class StackTests(test.TestCase):
         stack = self.stacks.first()
 
         api.heat.template_validate(IsA(http.HttpRequest),
-                                   template=template['data']) \
+                                   files={},
+                                   template=hc_format.parse(template['data'])) \
            .AndReturn(template['validate'])
 
         api.heat.stack_create(IsA(http.HttpRequest),
                               stack_name=stack.stack_name,
                               timeout_mins=60,
                               disable_rollback=True,
-                              template=template['data'],
+                              template=hc_format.parse(template['data']),
                               parameters={'param1': 'some string',
                                           'param2': 42,
                                           'param3': '{"key": "value"}',
                                           'param4': 'a,b,c',
                                           'param5': True},
-                              password='password')
+                              password='password',
+                              files={})
 
         self.mox.ReplayAll()
 
@@ -612,7 +621,8 @@ class StackTests(test.TestCase):
                            stack.id).AndReturn(stack)
         # POST template form, validation
         api.heat.template_validate(IsA(http.HttpRequest),
-                                   template=template.data) \
+                                   files={},
+                                   template=hc_format.parse(template.data)) \
            .AndReturn(json.loads(template.validate))
 
         # GET to edit form
@@ -631,8 +641,9 @@ class StackTests(test.TestCase):
             'disable_rollback': True,
             'timeout_mins': 61,
             'password': 'password',
-            'template': IsA(six.text_type),
+            'template': None,
-            'parameters': IsA(dict)
+            'parameters': IsA(dict),
+            'files': None
         }
         api.heat.stack_update(IsA(http.HttpRequest),
                               stack_id=stack.id,
@@ -755,15 +766,17 @@ class StackTests(test.TestCase):
         stack = self.stacks.first()
 
         api.heat.template_validate(IsA(http.HttpRequest),
-                                   template=template.data) \
+                                   files={},
+                                   template=hc_format.parse(template.data)) \
            .AndReturn(json.loads(template.validate))
 
         api.heat.stack_preview(IsA(http.HttpRequest),
                                stack_name=stack.stack_name,
                                timeout_mins=60,
                                disable_rollback=True,
-                               template=template.data,
+                               template=None,
-                               parameters=IsA(dict)).AndReturn(stack)
+                               parameters=IsA(dict),
+                               files=None).AndReturn(stack)
 
         self.mox.ReplayAll()
 

openstack_dashboard/dashboards/project/stacks/views.py

@@ -179,19 +179,12 @@ class CreateStackView(forms.ModalFormView):
 
     def get_initial(self):
         initial = {}
-        self.load_kwargs(initial)
+        if 'environment_data' in self.kwargs:
+            initial['environment_data'] = self.kwargs['environment_data']
         if 'parameters' in self.kwargs:
             initial['parameters'] = json.dumps(self.kwargs['parameters'])
         return initial
 
-    def load_kwargs(self, initial):
-        # load the "passed through" data from template form
-        for prefix in ('template', 'environment'):
-            for suffix in ('_data', '_url'):
-                key = prefix + suffix
-                if key in self.kwargs:
-                    initial[key] = self.kwargs[key]
-
     def get_form_kwargs(self):
         kwargs = super(CreateStackView, self).get_form_kwargs()
         if 'parameters' in self.kwargs:

openstack_dashboard/test/api_tests/heat_tests.py

@@ -9,10 +9,12 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import six
 
 from django.conf import settings
 from django.test.utils import override_settings  # noqa
 
+from horizon import exceptions
 from openstack_dashboard import api
 from openstack_dashboard.test import helpers as test
 
@@ -211,3 +213,84 @@ class HeatApiTests(test.APITestCase):
                                                **form_data)
         from heatclient.v1 import stacks
         self.assertIsInstance(returned_stack, stacks.Stack)
+
+    def test_get_template_files_with_template_data(self):
+        tmpl = '''
+    heat_template_version: 2013-05-23
+    resources:
+      server1:
+        type: OS::Nova::Server
+        properties:
+            flavor: m1.medium
+            image: cirros
+    '''
+        expected_files = {}
+        files = api.heat.get_template_files(template_data=tmpl)[0]
+        self.assertEqual(files, expected_files)
+
+    def test_get_template_files(self):
+        tmpl = '''
+    heat_template_version: 2013-05-23
+    resources:
+      server1:
+        type: OS::Nova::Server
+        properties:
+            flavor: m1.medium
+            image: cirros
+            user_data_format: RAW
+            user_data:
+              get_file: http://test.example/example
+    '''
+        expected_files = {u'http://test.example/example': b'echo "test"'}
+        url = 'http://test.example/example'
+        data = b'echo "test"'
+        self.mox.StubOutWithMock(six.moves.urllib.request, 'urlopen')
+        six.moves.urllib.request.urlopen(url).AndReturn(
+            six.BytesIO(data))
+        self.mox.ReplayAll()
+        files = api.heat.get_template_files(template_data=tmpl)[0]
+        self.assertEqual(files, expected_files)
+
+    def test_get_template_files_with_template_url(self):
+        url = 'https://test.example/example.yaml'
+        data = b'''
+    heat_template_version: 2013-05-23
+    resources:
+      server1:
+        type: OS::Nova::Server
+        properties:
+            flavor: m1.medium
+            image: cirros
+            user_data_format: RAW
+            user_data:
+              get_file: http://test.example/example
+    '''
+        url2 = 'http://test.example/example'
+        data2 = b'echo "test"'
+        expected_files = {'http://test.example/example': b'echo "test"'}
+        self.mox.StubOutWithMock(six.moves.urllib.request, 'urlopen')
+        six.moves.urllib.request.urlopen(url).AndReturn(
+            six.BytesIO(data))
+        six.moves.urllib.request.urlopen(url2).AndReturn(
+            six.BytesIO(data2))
+        self.mox.ReplayAll()
+        files = api.heat.get_template_files(template_url=url)[0]
+        self.assertEqual(files, expected_files)
+
+    def test_get_template_files_invalid(self):
+        tmpl = '''
+    heat_template_version: 2013-05-23
+    resources:
+      server1:
+        type: OS::Nova::Server
+        properties:
+            flavor: m1.medium
+            image: cirros
+            user_data_format: RAW
+            user_data:
+              get_file: file:///example
+    '''
+        try:
+            api.heat.get_template_files(template_data=tmpl)[0]
+        except exceptions.GetFileError:
+            self.assertRaises(exceptions.GetFileError)