From ca8e84f0c34f45a714fbfdc318ad7caf9eddfeaf Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Mon, 27 Mar 2017 13:22:50 +0100 Subject: [PATCH] Write user parameters environment to swift Currently we only merge this into the local merged environment, but instead we should write it as a separate user-environment in swift, which will enable heat server side parameter merging to work in future. The _write_user_environment function is added to handle writing the file to the local tmpdir tht_root, and uploading the same data to the plan. Partial-Bug: #1635409 Change-Id: I74bde48ef18c2e59233b7535578f394fbd1f7553 --- .../overcloud_deploy/test_overcloud_deploy.py | 93 ++++++++++--------- tripleoclient/v1/overcloud_deploy.py | 43 ++++++++- 2 files changed, 90 insertions(+), 46 deletions(-) diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py index 7a2874453..58ec79b76 100644 --- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py +++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py @@ -137,11 +137,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): 'StackAction': 'UPDATE', } - testcase = self - - def _custom_create_params_env(self, parameters): + def _custom_create_params_env(_self, parameters, tht_root, + container_name): for key, value in six.iteritems(parameters): - testcase.assertEqual(value, expected_parameters[key]) + self.assertEqual(value, expected_parameters[key]) parameter_defaults = {"parameter_defaults": parameters} return parameter_defaults @@ -168,8 +167,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_validate_args') - @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_create_parameters_env', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) @mock.patch('tripleoclient.utils.write_overcloudrc', autospec=True) @@ -179,29 +176,32 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): @mock.patch('heatclient.common.template_utils.get_template_contents', autospec=True) @mock.patch('uuid.uuid1', autospec=True) + @mock.patch('uuid.uuid4', autospec=True) @mock.patch('time.time', autospec=True) @mock.patch('shutil.copytree', autospec=True) @mock.patch('tempfile.mkdtemp', autospec=True) def test_tht_deploy(self, mock_tmpdir, mock_copy, mock_time, + mock_uuid4, mock_uuid1, mock_get_template_contents, wait_for_stack_ready_mock, mock_remove_known_hosts, mock_write_overcloudrc, mock_create_tempest_deployer_input, - mock_create_parameters_env, mock_validate_args, + mock_validate_args, mock_breakpoints_cleanup, mock_tarball, mock_postconfig, mock_get_overcloud_endpoint, mock_invoke_plan_env_wf): - arglist = ['--templates', '--ceph-storage-scale', '3'] + arglist = ['--templates', '--ceph-storage-scale', '3', '--no-cleanup'] verifylist = [ ('templates', '/usr/share/openstack-tripleo-heat-templates/'), ('ceph_storage_scale', 3) ] - mock_tmpdir.return_value = "/tmp/tht" + mock_tmpdir.return_value = self.tmp_dir.path mock_uuid1.return_value = "uuid" + mock_uuid4.return_value = "uuid4" mock_time.return_value = 123456789 clients = self.app.client_manager @@ -233,33 +233,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): baremetal = clients.baremetal baremetal.node.list.return_value = range(10) - expected_parameters = { - 'CephClusterFSID': 'uuid', - 'CephStorageCount': 3, - 'ExtraConfig': '{}', - 'HypervisorNeutronPhysicalBridge': 'br-ex', - 'HypervisorNeutronPublicInterface': 'nic1', - 'NeutronDnsmasqOptions': 'dhcp-option-force=26,1400', - 'NeutronFlatNetworks': 'datacentre', - 'NeutronNetworkType': 'gre', - 'NeutronPublicInterface': 'nic1', - 'NeutronTunnelTypes': 'gre', - 'NtpServer': '', - 'SnmpdReadonlyUserPassword': 'PASSWORD', - 'DeployIdentifier': 123456789, - 'UpdateIdentifier': '', - 'StackAction': 'CREATE', - } - - testcase = self - - def _custom_create_params_env(self, parameters): - for key, value in six.iteritems(parameters): - testcase.assertEqual(value, expected_parameters[key]) - parameter_defaults = {"parameter_defaults": parameters} - return parameter_defaults - - mock_create_parameters_env.side_effect = _custom_create_params_env + parameters_env = { + 'parameter_defaults': { + 'CephStorageCount': 3, + 'StackAction': 'CREATE', + 'UpdateIdentifier': ''}} self.cmd.take_action(parsed_args) @@ -274,11 +252,36 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_validate_args.assert_called_once_with(parsed_args) mock_tarball.create_tarball.assert_called_with( - '/tmp/tht/tripleo-heat-templates', mock.ANY) + self.tmp_dir.join('tripleo-heat-templates'), mock.ANY) mock_tarball.tarball_extract_to_swift_container.assert_called_with( clients.tripleoclient.object_store, mock.ANY, 'overcloud') self.assertFalse(mock_invoke_plan_env_wf.called) + calls = [ + mock.call('overcloud', + 'user-environments/tripleoclient-parameters.yaml', + yaml.safe_dump(parameters_env, + default_flow_style=False)), + mock.call('overcloud', + 'user-environment.yaml', + yaml.safe_dump({}, default_flow_style=False)), + mock.call('overcloud', + 'plan-environment.yaml', + yaml.safe_dump({'environments': + [{'path': 'user-environment.yaml'}]}, + default_flow_style=False))] + + object_client = clients.tripleoclient.object_store + object_client.put_object.assert_has_calls(calls) + tmp_param_env = self.tmp_dir.join( + 'tripleo-heat-templates', + 'user-environments/tripleoclient-parameters.yaml') + self.assertTrue(os.path.isfile(tmp_param_env)) + with open(tmp_param_env, 'r') as f: + env_map = yaml.safe_load(f) + self.assertEqual(env_map.get('parameter_defaults'), + parameters_env.get('parameter_defaults')) + @mock.patch('tripleoclient.workflows.parameters.invoke_plan_env_workflows', autospec=True) @mock.patch('shutil.rmtree', autospec=True) @@ -377,7 +380,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): testcase = self - def _custom_create_params_env(self, parameters): + def _custom_create_params_env(_self, parameters, tht_root, + container_name): for key, value in six.iteritems(parameters): testcase.assertEqual(value, expected_parameters[key]) parameter_defaults = {"parameter_defaults": parameters} @@ -491,7 +495,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): testcase = self - def _custom_create_params_env(self, parameters): + def _custom_create_params_env(_self, parameters, tht_root, + container_name): testcase.assertTrue('DeployIdentifier' not in parameters) parameter_defaults = {"parameter_defaults": parameters} return parameter_defaults @@ -1165,7 +1170,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): @mock.patch('tripleoclient.workflows.plan_management.tarball', autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_create_parameters_env') + '_create_parameters_env', autospec=True) @mock.patch('tripleoclient.utils.write_overcloudrc') @mock.patch('heatclient.common.template_utils.' 'process_environment_and_files', autospec=True) @@ -1195,7 +1200,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_env = yaml.safe_dump({'environments': []}) object_client.get_object.return_value = ({}, mock_env) - def _custom_create_params_env(parameters): + def _custom_create_params_env(_self, parameters, tht_root, + container_name): parameter_defaults = {"parameter_defaults": parameters} return parameter_defaults @@ -1219,7 +1225,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_validate_args') @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_create_parameters_env') + '_create_parameters_env', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) @mock.patch('tripleoclient.utils.write_overcloudrc') @@ -1312,7 +1318,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): 'NtpServer': 'ntp', } - def _custom_create_params_env(parameters): + def _custom_create_params_env(_self, parameters, tht_root, + container_name): for key, value in six.iteritems(parameters): self.assertEqual(value, expected_parameters[key]) parameter_defaults = {"parameter_defaults": parameters} diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index d637b7316..9c4e87b45 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -121,10 +121,44 @@ class DeployOvercloud(command.Command): 'rhel_reg_activation_key': args.reg_activation_key} return [registry], {"parameter_defaults": user_env} - def _create_parameters_env(self, parameters): + def _create_parameters_env(self, parameters, tht_root, container_name): parameter_defaults = {"parameter_defaults": parameters} + env_path, swift_path = self._write_user_environment( + parameter_defaults, + 'tripleoclient-parameters.yaml', + tht_root, + container_name) return parameter_defaults + def _write_user_environment(self, env_map, abs_env_path, tht_root, + container_name): + # We write the env_map to the local /tmp tht_root and also + # to the swift plan container. + contents = yaml.safe_dump(env_map, default_flow_style=False) + env_dirname = os.path.dirname(abs_env_path) + user_env_dir = os.path.join( + tht_root, 'user-environments', env_dirname[1:]) + user_env_path = os.path.join( + user_env_dir, os.path.basename(abs_env_path)) + self.log.debug("user_env_path=%s" % user_env_path) + if not os.path.exists(user_env_dir): + os.makedirs(user_env_dir) + with open(user_env_path, 'w') as f: + self.log.debug("Writing user environment %s" % user_env_path) + f.write(contents) + + # Upload to swift + if abs_env_path.startswith("/"): + swift_path = "user-environments/{}".format(abs_env_path[1:]) + else: + swift_path = "user-environments/{}".format(abs_env_path) + contents = yaml.safe_dump(env_map, default_flow_style=False) + self.log.debug("Uploading %s to swift at %s" + % (abs_env_path, swift_path)) + self.object_client.put_object(container_name, swift_path, contents) + + return user_env_path, swift_path + def _process_multiple_environments(self, created_env_files, tht_root, user_tht_root, cleanup=True): env_files = {} @@ -290,7 +324,7 @@ class DeployOvercloud(command.Command): self.workflow_client, container=container_name, parameters=params) - contents = yaml.safe_dump(env) + contents = yaml.safe_dump(env, default_flow_style=False) # Until we have a well defined plan update workflow in tripleo-common # we need to manually add an environment in swift and for users @@ -419,7 +453,10 @@ class DeployOvercloud(command.Command): if parsed_args.environment_directories: created_env_files.extend(self._load_environment_directories( parsed_args.environment_directories)) - env.update(self._create_parameters_env(parameters)) + + env.update(self._create_parameters_env(parameters, + tht_root, + parsed_args.stack)) if parsed_args.rhel_reg: reg_env_files, reg_env = self._create_registration_env(parsed_args)