Write user parameters environment to swift

Currently we only merge this into the local merged environment,
but instead we should write it as a separate user-environment in
swift, which will enable heat server side parameter merging to
work in future.  The _write_user_environment function is added to
handle writing the file to the local tmpdir tht_root, and uploading
the same data to the plan.

Partial-Bug: #1635409
Change-Id: I74bde48ef18c2e59233b7535578f394fbd1f7553

tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py

@@ -137,11 +137,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             'StackAction': 'UPDATE',
         }
 
-        testcase = self
+        def _custom_create_params_env(_self, parameters, tht_root,
-
+                                      container_name):
-        def _custom_create_params_env(self, parameters):
             for key, value in six.iteritems(parameters):
-                testcase.assertEqual(value, expected_parameters[key])
+                self.assertEqual(value, expected_parameters[key])
             parameter_defaults = {"parameter_defaults": parameters}
             return parameter_defaults
 
@@ -168,8 +167,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
                 autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_validate_args')
-    @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_create_parameters_env', autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
     @mock.patch('tripleoclient.utils.write_overcloudrc', autospec=True)
@@ -179,29 +176,32 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     @mock.patch('heatclient.common.template_utils.get_template_contents',
                 autospec=True)
     @mock.patch('uuid.uuid1', autospec=True)
+    @mock.patch('uuid.uuid4', autospec=True)
     @mock.patch('time.time', autospec=True)
     @mock.patch('shutil.copytree', autospec=True)
     @mock.patch('tempfile.mkdtemp', autospec=True)
     def test_tht_deploy(self, mock_tmpdir, mock_copy, mock_time,
+                        mock_uuid4,
                         mock_uuid1,
                         mock_get_template_contents,
                         wait_for_stack_ready_mock,
                         mock_remove_known_hosts,
                         mock_write_overcloudrc,
                         mock_create_tempest_deployer_input,
-                        mock_create_parameters_env, mock_validate_args,
+                        mock_validate_args,
                         mock_breakpoints_cleanup, mock_tarball,
                         mock_postconfig, mock_get_overcloud_endpoint,
                         mock_invoke_plan_env_wf):
 
-        arglist = ['--templates', '--ceph-storage-scale', '3']
+        arglist = ['--templates', '--ceph-storage-scale', '3', '--no-cleanup']
         verifylist = [
             ('templates', '/usr/share/openstack-tripleo-heat-templates/'),
             ('ceph_storage_scale', 3)
         ]
 
-        mock_tmpdir.return_value = "/tmp/tht"
+        mock_tmpdir.return_value = self.tmp_dir.path
         mock_uuid1.return_value = "uuid"
+        mock_uuid4.return_value = "uuid4"
         mock_time.return_value = 123456789
 
         clients = self.app.client_manager
@@ -233,33 +233,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         baremetal = clients.baremetal
         baremetal.node.list.return_value = range(10)
 
-        expected_parameters = {
+        parameters_env = {
-            'CephClusterFSID': 'uuid',
+            'parameter_defaults': {
-            'CephStorageCount': 3,
+                'CephStorageCount': 3,
-            'ExtraConfig': '{}',
+                'StackAction': 'CREATE',
-            'HypervisorNeutronPhysicalBridge': 'br-ex',
+                'UpdateIdentifier': ''}}
-            'HypervisorNeutronPublicInterface': 'nic1',
-            'NeutronDnsmasqOptions': 'dhcp-option-force=26,1400',
-            'NeutronFlatNetworks': 'datacentre',
-            'NeutronNetworkType': 'gre',
-            'NeutronPublicInterface': 'nic1',
-            'NeutronTunnelTypes': 'gre',
-            'NtpServer': '',
-            'SnmpdReadonlyUserPassword': 'PASSWORD',
-            'DeployIdentifier': 123456789,
-            'UpdateIdentifier': '',
-            'StackAction': 'CREATE',
-        }
-
-        testcase = self
-
-        def _custom_create_params_env(self, parameters):
-            for key, value in six.iteritems(parameters):
-                testcase.assertEqual(value, expected_parameters[key])
-            parameter_defaults = {"parameter_defaults": parameters}
-            return parameter_defaults
-
-        mock_create_parameters_env.side_effect = _custom_create_params_env
 
         self.cmd.take_action(parsed_args)
 
@@ -274,11 +252,36 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_validate_args.assert_called_once_with(parsed_args)
 
         mock_tarball.create_tarball.assert_called_with(
-            '/tmp/tht/tripleo-heat-templates', mock.ANY)
+            self.tmp_dir.join('tripleo-heat-templates'), mock.ANY)
         mock_tarball.tarball_extract_to_swift_container.assert_called_with(
             clients.tripleoclient.object_store, mock.ANY, 'overcloud')
         self.assertFalse(mock_invoke_plan_env_wf.called)
 
+        calls = [
+            mock.call('overcloud',
+                      'user-environments/tripleoclient-parameters.yaml',
+                      yaml.safe_dump(parameters_env,
+                                     default_flow_style=False)),
+            mock.call('overcloud',
+                      'user-environment.yaml',
+                      yaml.safe_dump({}, default_flow_style=False)),
+            mock.call('overcloud',
+                      'plan-environment.yaml',
+                      yaml.safe_dump({'environments':
+                                      [{'path': 'user-environment.yaml'}]},
+                                     default_flow_style=False))]
+
+        object_client = clients.tripleoclient.object_store
+        object_client.put_object.assert_has_calls(calls)
+        tmp_param_env = self.tmp_dir.join(
+            'tripleo-heat-templates',
+            'user-environments/tripleoclient-parameters.yaml')
+        self.assertTrue(os.path.isfile(tmp_param_env))
+        with open(tmp_param_env, 'r') as f:
+            env_map = yaml.safe_load(f)
+        self.assertEqual(env_map.get('parameter_defaults'),
+                         parameters_env.get('parameter_defaults'))
+
     @mock.patch('tripleoclient.workflows.parameters.invoke_plan_env_workflows',
                 autospec=True)
     @mock.patch('shutil.rmtree', autospec=True)
@@ -377,7 +380,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         testcase = self
 
-        def _custom_create_params_env(self, parameters):
+        def _custom_create_params_env(_self, parameters, tht_root,
+                                      container_name):
             for key, value in six.iteritems(parameters):
                 testcase.assertEqual(value, expected_parameters[key])
             parameter_defaults = {"parameter_defaults": parameters}
@@ -491,7 +495,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
 
         testcase = self
 
-        def _custom_create_params_env(self, parameters):
+        def _custom_create_params_env(_self, parameters, tht_root,
+                                      container_name):
             testcase.assertTrue('DeployIdentifier' not in parameters)
             parameter_defaults = {"parameter_defaults": parameters}
             return parameter_defaults
@@ -1165,7 +1170,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     @mock.patch('tripleoclient.workflows.plan_management.tarball',
                 autospec=True)
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_create_parameters_env')
+                '_create_parameters_env', autospec=True)
     @mock.patch('tripleoclient.utils.write_overcloudrc')
     @mock.patch('heatclient.common.template_utils.'
                 'process_environment_and_files', autospec=True)
@@ -1195,7 +1200,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
         mock_env = yaml.safe_dump({'environments': []})
         object_client.get_object.return_value = ({}, mock_env)
 
-        def _custom_create_params_env(parameters):
+        def _custom_create_params_env(_self, parameters, tht_root,
+                                      container_name):
             parameter_defaults = {"parameter_defaults": parameters}
             return parameter_defaults
 
@@ -1219,7 +1225,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
                 '_validate_args')
     @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
-                '_create_parameters_env')
+                '_create_parameters_env', autospec=True)
     @mock.patch('tripleoclient.utils.create_tempest_deployer_input',
                 autospec=True)
     @mock.patch('tripleoclient.utils.write_overcloudrc')
@@ -1312,7 +1318,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
             'NtpServer': 'ntp',
         }
 
-        def _custom_create_params_env(parameters):
+        def _custom_create_params_env(_self, parameters, tht_root,
+                                      container_name):
             for key, value in six.iteritems(parameters):
                 self.assertEqual(value, expected_parameters[key])
             parameter_defaults = {"parameter_defaults": parameters}

tripleoclient/v1/overcloud_deploy.py

@@ -121,10 +121,44 @@ class DeployOvercloud(command.Command):
                     'rhel_reg_activation_key': args.reg_activation_key}
         return [registry], {"parameter_defaults": user_env}
 
-    def _create_parameters_env(self, parameters):
+    def _create_parameters_env(self, parameters, tht_root, container_name):
         parameter_defaults = {"parameter_defaults": parameters}
+        env_path, swift_path = self._write_user_environment(
+            parameter_defaults,
+            'tripleoclient-parameters.yaml',
+            tht_root,
+            container_name)
         return parameter_defaults
 
+    def _write_user_environment(self, env_map, abs_env_path, tht_root,
+                                container_name):
+        # We write the env_map to the local /tmp tht_root and also
+        # to the swift plan container.
+        contents = yaml.safe_dump(env_map, default_flow_style=False)
+        env_dirname = os.path.dirname(abs_env_path)
+        user_env_dir = os.path.join(
+            tht_root, 'user-environments', env_dirname[1:])
+        user_env_path = os.path.join(
+            user_env_dir, os.path.basename(abs_env_path))
+        self.log.debug("user_env_path=%s" % user_env_path)
+        if not os.path.exists(user_env_dir):
+            os.makedirs(user_env_dir)
+        with open(user_env_path, 'w') as f:
+            self.log.debug("Writing user environment %s" % user_env_path)
+            f.write(contents)
+
+        # Upload to swift
+        if abs_env_path.startswith("/"):
+            swift_path = "user-environments/{}".format(abs_env_path[1:])
+        else:
+            swift_path = "user-environments/{}".format(abs_env_path)
+        contents = yaml.safe_dump(env_map, default_flow_style=False)
+        self.log.debug("Uploading %s to swift at %s"
+                       % (abs_env_path, swift_path))
+        self.object_client.put_object(container_name, swift_path, contents)
+
+        return user_env_path, swift_path
+
     def _process_multiple_environments(self, created_env_files, tht_root,
                                        user_tht_root, cleanup=True):
         env_files = {}
@@ -290,7 +324,7 @@ class DeployOvercloud(command.Command):
                 self.workflow_client, container=container_name,
                 parameters=params)
 
-        contents = yaml.safe_dump(env)
+        contents = yaml.safe_dump(env, default_flow_style=False)
 
         # Until we have a well defined plan update workflow in tripleo-common
         # we need to manually add an environment in swift and for users
@@ -419,7 +453,10 @@ class DeployOvercloud(command.Command):
         if parsed_args.environment_directories:
             created_env_files.extend(self._load_environment_directories(
                 parsed_args.environment_directories))
-        env.update(self._create_parameters_env(parameters))
+
+        env.update(self._create_parameters_env(parameters,
+                                               tht_root,
+                                               parsed_args.stack))
 
         if parsed_args.rhel_reg:
             reg_env_files, reg_env = self._create_registration_env(parsed_args)