Hook into environment merging on server

If environment_files is specified, look in the files dict for each
specified environment file and merge into the stack's environment. This
is the same workflow that previously occurred client-side.

Change-Id: Ibe46fd35de7988920c101a83259c06c8f8a3ed0b
Implements: blueprint multi-environments

heat/engine/service.py

@@ -69,6 +69,8 @@ from heat.objects import watch_data
 from heat.objects import watch_rule
 from heat.rpc import api as rpc_api
 from heat.rpc import worker_api as rpc_worker_api
+from heatclient.common import environment_format
+from heatclient.common import template_utils
 
 cfg.CONF.import_opt('engine_life_check_timeout', 'heat.common.config')
 cfg.CONF.import_opt('max_resources_per_stack', 'heat.common.config')
@@ -621,7 +623,8 @@ class EngineService(service.Service):
             raise exception.RequestLimitExceeded(message=message)
 
     def _parse_template_and_validate_stack(self, cnxt, stack_name, template,
-                                           params, files, args, owner_id=None,
+                                           params, files, environment_files,
+                                           args, owner_id=None,
                                            nested_depth=0, user_creds_id=None,
                                            stack_user_project_id=None,
                                            convergence=False,
@@ -641,6 +644,7 @@ class EngineService(service.Service):
             new_params.update(params.get(rpc_api.STACK_PARAMETERS, {}))
             params[rpc_api.STACK_PARAMETERS] = new_params
 
+        self._merge_environments(environment_files, files, params)
         env = environment.Environment(params)
 
         tmpl = templatem.Template(template, files=files, env=env)
@@ -662,6 +666,30 @@ class EngineService(service.Service):
             env.registry.log_resource_info(prefix=stack_name)
         return stack
 
+    @staticmethod
+    def _merge_environments(environment_files, files, params):
+        """Merges environment files into the stack input parameters.
+
+        If a list of environment files have been specified, this call will
+        pull the contents of each from the files dict, parse them as
+        environments, and merge them into the stack input params. This
+        behavior is the same as earlier versions of the Heat client that
+        performed this params population client-side.
+
+        :param environment_files: ordered names of the environment files
+               found in the files dict
+        :type  environment_files: list or None
+        :param files: mapping of stack filenames to contents
+        :type  files: dict
+        :param params: parameters describing the stack
+        :type  dict:
+        """
+        if environment_files:
+            for filename in environment_files:
+                raw_env = files[filename]
+                parsed_env = environment_format.parse(raw_env)
+                template_utils.deep_update(params, parsed_env)
+
     @context.request_context
     def preview_stack(self, cnxt, stack_name, template, params, files,
                       args, environment_files=None):
@@ -689,6 +717,7 @@ class EngineService(service.Service):
                                                         template,
                                                         params,
                                                         files,
+                                                        environment_files,
                                                         args,
                                                         convergence=conv_eng)
 
@@ -751,9 +780,9 @@ class EngineService(service.Service):
         convergence = cfg.CONF.convergence_engine
 
         stack = self._parse_template_and_validate_stack(
-            cnxt, stack_name, template, params, files, args, owner_id,
+            cnxt, stack_name, template, params, files, environment_files,
-            nested_depth, user_creds_id, stack_user_project_id, convergence,
+            args, owner_id, nested_depth, user_creds_id,
-            parent_resource_name)
+            stack_user_project_id, convergence, parent_resource_name)
 
         self.resource_enforcer.enforce_stack(stack)
         stack_id = stack.store()
@@ -786,6 +815,7 @@ class EngineService(service.Service):
         :param files: Files referenced from the template
         :param args: Request parameters/args passed from API
         """
+
         # Now parse the template and any parameters for the updated
         # stack definition.  If PARAM_EXISTING is specified, we merge
         # any environment provided into the existing one and attempt
@@ -877,6 +907,9 @@ class EngineService(service.Service):
                names included in the files dict
         :type  environment_files: list or None
         """
+        # Handle server-side environment file resolution
+        self._merge_environments(environment_files, files, params)
+
         # Get the database representation of the existing stack
         db_stack = self._get_stack(cnxt, stack_identity)
         LOG.info(_LI('Updating stack %s'), db_stack.name)
@@ -930,6 +963,9 @@ class EngineService(service.Service):
         Note that at this stage the template has already been fetched from the
         heat-api process if using a template-url.
         """
+        # Handle server-side environment file resolution
+        self._merge_environments(environment_files, files, params)
+
         # Get the database representation of the existing stack
         db_stack = self._get_stack(cnxt, stack_identity)
         LOG.info(_LI('Previewing update of stack %s'), db_stack.name)
@@ -1104,6 +1140,7 @@ class EngineService(service.Service):
 
             service_check_defer = True
 
+        self._merge_environments(environment_files, files, params)
         env = environment.Environment(params)
         tmpl = templatem.Template(template, files=files, env=env)
         try:

heat/tests/engine/service/test_service_engine.py

@@ -411,3 +411,50 @@ class ServiceEngineTest(common.HeatTestCase):
         self.eng.start()
         self.assertEqual(cfg.CONF.executor_thread_pool_size,
                          cfg.CONF.database.max_overflow)
+
+    def test_merge_environments(self):
+        # Setup
+        params = {'parameters': {
+            'p0': 'CORRECT',
+            'p1': 'INCORRECT',
+            'p2': 'INCORRECT'}
+        }
+        env_1 = '''
+        {'parameters' : {
+            'p1': 'CORRECT',
+            'p2': 'INCORRECT-ENV-1',
+        }}'''
+        env_2 = '''
+        {'parameters': {
+            'p2': 'CORRECT'
+        }}'''
+
+        files = {'env_1': env_1, 'env_2': env_2}
+        environment_files = ['env_1', 'env_2']
+
+        # Test
+        self.eng._merge_environments(environment_files, files, params)
+
+        # Verify
+        expected = {'parameters': {
+            'p0': 'CORRECT',
+            'p1': 'CORRECT',
+            'p2': 'CORRECT',
+        }}
+        self.assertEqual(expected, params)
+
+    def test_merge_environments_no_env_files(self):
+        params = {'parameters': {'p0': 'CORRECT'}}
+        env_1 = '''
+        {'parameters' : {
+            'p0': 'INCORRECT',
+        }}'''
+
+        files = {'env_1': env_1}
+
+        # Test - Should ignore env_1 in files
+        self.eng._merge_environments(None, files, params)
+
+        # Verify
+        expected = {'parameters': {'p0': 'CORRECT'}}
+        self.assertEqual(expected, params)

heat/tests/engine/service/test_stack_create.py

@@ -41,7 +41,8 @@ class StackCreateTest(common.HeatTestCase):
 
     @mock.patch.object(threadgroup, 'ThreadGroup')
     @mock.patch.object(stack.Stack, 'validate')
-    def _test_stack_create(self, stack_name, mock_validate, mock_tg):
+    def _test_stack_create(self, stack_name, mock_validate, mock_tg,
+                           environment_files=None):
         mock_tg.return_value = tools.DummyThreadGroup()
 
         params = {'foo': 'bar'}
@@ -53,8 +54,10 @@ class StackCreateTest(common.HeatTestCase):
         mock_env = self.patchobject(environment, 'Environment',
                                     return_value=stk.env)
         mock_stack = self.patchobject(stack, 'Stack', return_value=stk)
+        mock_merge = self.patchobject(self.man, '_merge_environments')
         result = self.man.create_stack(self.ctx, stack_name,
-                                       template, params, None, {})
+                                       template, params, None, {},
+                                       environment_files=environment_files)
         self.assertEqual(stk.identifier(), result)
         self.assertIsInstance(result, dict)
         self.assertTrue(result['stack_id'])
@@ -67,12 +70,21 @@ class StackCreateTest(common.HeatTestCase):
                                            stack_user_project_id=None,
                                            convergence=False,
                                            parent_resource=None)
+
+        if environment_files:
+            mock_merge.assert_called_once_with(environment_files, None, params)
         mock_validate.assert_called_once_with()
 
     def test_stack_create(self):
         stack_name = 'service_create_test_stack'
         self._test_stack_create(stack_name)
 
+    def test_stack_create_with_environment_files(self):
+        stack_name = 'env_files_test_stack'
+        environment_files = ['env_1', 'env_2']
+        self._test_stack_create(stack_name,
+                                environment_files=environment_files)
+
     def test_stack_create_equals_max_per_tenant(self):
         cfg.CONF.set_override('max_stacks_per_tenant', 1)
         stack_name = 'service_create_test_stack_equals_max'

heat/tests/engine/service/test_stack_update.py

@@ -96,6 +96,37 @@ class ServiceStackUpdateTest(common.HeatTestCase):
         mock_load.assert_called_once_with(self.ctx, stack=s)
         mock_validate.assert_called_once_with()
 
+    def test_stack_update_with_environment_files(self):
+        # Setup
+        stack_name = 'service_update_env_files_stack'
+        params = {}
+        template = '{ "Template": "data" }'
+        old_stack = tools.get_stack(stack_name, self.ctx)
+        sid = old_stack.store()
+        old_stack.set_stack_user_project_id('1234')
+        stack_object.Stack.get_by_id(self.ctx, sid)
+
+        stk = tools.get_stack(stack_name, self.ctx)
+
+        # prepare mocks
+        self.patchobject(stack, 'Stack', return_value=stk)
+        self.patchobject(stack.Stack, 'load', return_value=old_stack)
+        self.patchobject(templatem, 'Template', return_value=stk.t)
+        self.patchobject(environment, 'Environment', return_value=stk.env)
+        self.patchobject(stk, 'validate', return_value=None)
+        self.patchobject(grevent, 'Event', return_value=mock.Mock())
+
+        mock_merge = self.patchobject(self.man, '_merge_environments')
+
+        # Test
+        environment_files = ['env_1']
+        self.man.update_stack(self.ctx, old_stack.identifier(),
+                              template, params, None, {},
+                              environment_files=environment_files)
+
+        # Verify
+        mock_merge.assert_called_once_with(environment_files, None, params)
+
     def test_stack_update_existing_parameters(self):
         # Use a template with existing parameters, then update the stack
         # with a template containing additional parameters and ensure all
@@ -712,7 +743,8 @@ resources:
         self.man = service.EngineService('a-host', 'a-topic')
         self.man.thread_group_mgr = tools.DummyThreadGroupManager()
 
-    def _test_stack_update_preview(self, orig_template, new_template):
+    def _test_stack_update_preview(self, orig_template, new_template,
+                                   environment_files=None):
         stack_name = 'service_update_test_stack_preview'
         params = {'foo': 'bar'}
         old_stack = tools.get_stack(stack_name, self.ctx,
@@ -731,6 +763,7 @@ resources:
         mock_env = self.patchobject(environment, 'Environment',
                                     return_value=stk.env)
         mock_validate = self.patchobject(stk, 'validate', return_value=None)
+        mock_merge = self.patchobject(self.man, '_merge_environments')
 
         # Patch _resolve_all_attributes or it tries to call novaclient
         self.patchobject(resource.Resource, '_resolve_all_attributes',
@@ -738,10 +771,13 @@ resources:
 
         # do preview_update_stack
         api_args = {'timeout_mins': 60}
-        result = self.man.preview_update_stack(self.ctx,
+        result = self.man.preview_update_stack(
-                                               old_stack.identifier(),
+            self.ctx,
-                                               new_template, params, None,
+            old_stack.identifier(),
-                                               api_args)
+            new_template, params, None,
+            api_args,
+            environment_files=environment_files)
+
         # assertions
         mock_stack.assert_called_once_with(
             self.ctx, stk.name, stk.t, convergence=False,
@@ -757,6 +793,9 @@ resources:
         mock_env.assert_called_once_with(params)
         mock_validate.assert_called_once_with()
 
+        if environment_files:
+            mock_merge.assert_called_once_with(environment_files, None, params)
+
         return result
 
     def test_stack_update_preview_added_unchanged(self):
@@ -824,3 +863,13 @@ resources:
         for section in empty_sections:
             section_contents = [x for x in result[section]]
             self.assertEqual([], section_contents)
+
+    def test_stack_update_preview_with_environment_files(self):
+        # Setup
+        environment_files = ['env_1']
+
+        # Test
+        self._test_stack_update_preview(self.old_tmpl, self.new_tmpl,
+                                        environment_files=environment_files)
+
+        # Assertions done in _test_stack_update_preview

heat/tests/test_engine_service.py

@@ -1123,7 +1123,7 @@ class StackServiceTest(common.HeatTestCase):
 
         self.assertEqual(0, len(sl))
 
-    def _preview_stack(self):
+    def _preview_stack(self, environment_files=None):
         res._register_class('GenericResource1', generic_rsrc.GenericResource)
         res._register_class('GenericResource2', generic_rsrc.GenericResource)
 
@@ -1138,7 +1138,8 @@ class StackServiceTest(common.HeatTestCase):
                    'SampleResource2': {'Type': 'GenericResource2'}}}
 
         return self.eng.preview_stack(self.ctx, stack_name, tpl,
-                                      params, files, args)
+                                      params, files, args,
+                                      environment_files=environment_files)
 
     def test_preview_stack_returns_a_stack(self):
         stack = self._preview_stack()
@@ -1181,6 +1182,17 @@ class StackServiceTest(common.HeatTestCase):
                                self._preview_stack)
         self.assertEqual(exception.StackValidationFailed, ex.exc_info[0])
 
+    @mock.patch.object(service.EngineService, '_merge_environments')
+    def test_preview_environment_files(self, mock_merge):
+        # Setup
+        environment_files = ['env_1']
+
+        # Test
+        self._preview_stack(environment_files=environment_files)
+
+        # Verify
+        mock_merge.assert_called_once_with(environment_files, None, {})
+
     @mock.patch.object(stack_object.Stack, 'get_by_name')
     def test_validate_new_stack_checks_existing_stack(self, mock_stack_get):
         mock_stack_get.return_value = 'existing_db_stack'
@@ -1389,13 +1401,13 @@ class StackServiceTest(common.HeatTestCase):
         # get parameters from adopt stack data which doesn't have it.
         args = {"adopt_stack_data": '''{}'''}
         self.eng._parse_template_and_validate_stack(
-            self.ctx, 'stack_name', template, {}, {}, args)
+            self.ctx, 'stack_name', template, {}, {}, None, args)
 
         args = {"adopt_stack_data": '''{
             "environment": {}
         }'''}
         self.eng._parse_template_and_validate_stack(
-            self.ctx, 'stack_name', template, {}, {}, args)
+            self.ctx, 'stack_name', template, {}, {}, None, args)
 
     def test_parse_adopt_stack_data_with_parameters(self):
         cfg.CONF.set_override('enable_stack_adopt', True)
@@ -1420,5 +1432,5 @@ class StackServiceTest(common.HeatTestCase):
                 }
             }}'''}
         stack = self.eng._parse_template_and_validate_stack(
-            self.ctx, 'stack_name', template, {}, {}, args)
+            self.ctx, 'stack_name', template, {}, {}, None, args)
         self.assertEqual(1, stack.parameters['volsize'])

heat_integrationtests/common/test.py

@@ -475,7 +475,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
     def stack_create(self, stack_name=None, template=None, files=None,
                      parameters=None, environment=None, tags=None,
                      expected_status='CREATE_COMPLETE',
-                     disable_rollback=True, enable_cleanup=True):
+                     disable_rollback=True, enable_cleanup=True,
+                     environment_files=None):
         name = stack_name or self._stack_rand_name()
         templ = template or self.template
         templ_files = files or {}
@@ -488,7 +489,8 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
             disable_rollback=disable_rollback,
             parameters=params,
             environment=env,
-            tags=tags
+            tags=tags,
+            environment_files=environment_files
         )
         if expected_status not in ['ROLLBACK_COMPLETE'] and enable_cleanup:
             self.addCleanup(self._stack_delete, name)

heat_integrationtests/functional/test_env_merge.py

@@ -0,0 +1,95 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from heat_integrationtests.functional import functional_base
+
+
+TEMPLATE = '''
+    heat_template_version: 2015-04-30
+    parameters:
+      p0:
+        type: string
+        default: CORRECT
+      p1:
+        type: string
+        default: INCORRECT
+      p2:
+        type: string
+        default: INCORRECT
+    resources:
+      r1:
+        type: test::R1
+      r2:
+        type: test::R2
+      r3a:
+        type: test::R3
+      r3b:
+        type: test::R3
+'''
+
+ENV_1 = '''
+    parameters:
+      p1: CORRECT
+      p2: INCORRECT-E1
+    resource_registry:
+      test::R1: OS::Heat::RandomString
+      test::R2: BROKEN
+      test::R3: OS::Heat::None
+'''
+
+ENV_2 = '''
+    parameters:
+      p2: CORRECT
+    resource_registry:
+      test::R2: OS::Heat::RandomString
+      resources:
+        r3b:
+          test::R3: OS::Heat::RandomString
+'''
+
+
+class EnvironmentMergingTests(functional_base.FunctionalTestsBase):
+
+    def test_server_environment_merging(self):
+
+        # Setup
+        files = {'env1.yaml': ENV_1, 'env2.yaml': ENV_2}
+        environment_files = ['env1.yaml', 'env2.yaml']
+
+        # Test
+        stack_id = self.stack_create(stack_name='env_merge',
+                                     template=TEMPLATE,
+                                     files=files,
+                                     environment_files=environment_files)
+
+        # Verify
+
+        # Since there is no environment show, the registry overriding
+        # is partially verified by there being no error. If it wasn't
+        # working, test::R2 would remain mapped to BROKEN in env1.
+
+        # Sanity check
+        resources = self.list_resources(stack_id)
+        self.assertEqual(4, len(resources))
+
+        # Verify the parameters are correctly set
+        stack = self.client.stacks.get(stack_id)
+        self.assertEqual('CORRECT', stack.parameters['p0'])
+        self.assertEqual('CORRECT', stack.parameters['p1'])
+        self.assertEqual('CORRECT', stack.parameters['p2'])
+
+        # Verify that r3b has been overridden into a RandomString
+        # by checking to see that it has a value
+        r3b = self.client.resources.get(stack_id, 'r3b')
+        r3b_attrs = r3b.attributes
+        self.assertTrue('value' in r3b_attrs)