diff --git a/tripleoclient/tests/fakes.py b/tripleoclient/tests/fakes.py index 6c244fe9b..5e92f5622 100644 --- a/tripleoclient/tests/fakes.py +++ b/tripleoclient/tests/fakes.py @@ -39,9 +39,24 @@ class FakeClientManager(object): self.tripleoclient = None self.auth_ref = None self.tripleoclient = FakeClientWrapper() + self.workflow_engine = mock.Mock() + + +class FakeWebSocket(object): + + def wait_for_message(self, execution_id): + return { + 'status': 'SUCCESS' + } + + def __enter__(self): + return self + + def __exit__(self, *args): + return class FakeClientWrapper(object): - def messaging_websocket(self, queue_name='tripleo'): - return mock.MagicMock() + def messaging_websocket(self, queue_name): + return FakeWebSocket() diff --git a/tripleoclient/tests/v1/overcloud_deploy/fakes.py b/tripleoclient/tests/v1/overcloud_deploy/fakes.py index c24de08be..c89bb7b6d 100644 --- a/tripleoclient/tests/v1/overcloud_deploy/fakes.py +++ b/tripleoclient/tests/v1/overcloud_deploy/fakes.py @@ -16,6 +16,8 @@ import mock from openstackclient.tests import utils +from tripleoclient.tests import fakes + FAKE_STACK = { 'parameters': { @@ -105,6 +107,10 @@ class FakeClientWrapper(object): def __init__(self): self._instance = mock.Mock() + self.object_store = mock.Mock() + + def messaging_websocket(self, queue_name): + return fakes.FakeWebSocket() class TestDeployOvercloud(utils.TestCommand): @@ -119,4 +125,5 @@ class TestDeployOvercloud(utils.TestCommand): self.app.client_manager.image = mock.Mock() self.app.client_manager.network = mock.Mock() self.app.client_manager.orchestration = mock.Mock() + self.app.client_manager.workflow_engine = mock.Mock() self.app.client_manager.tripleoclient = FakeClientWrapper() diff --git a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py index d1c89fbf7..e6873f435 100644 --- a/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py +++ b/tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py @@ -16,6 +16,7 @@ import fixtures import json import os +import shutil import six import tempfile import yaml @@ -58,16 +59,20 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): super(TestDeployOvercloud, self).tearDown() os.unlink(self.parameter_defaults_env_file) + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) @mock.patch("heatclient.common.event_utils.get_events") - @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env') + @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env', + autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_create_parameters_env') + '_create_parameters_env', autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_deploy_postconfig') + '_deploy_postconfig', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) - @mock.patch('tripleoclient.utils.generate_overcloud_passwords') - @mock.patch('tripleoclient.utils.create_overcloudrc') + @mock.patch('tripleoclient.utils.generate_overcloud_passwords', + autospec=True) + @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True) @mock.patch('os_cloud_config.keystone.setup_endpoints', autospec=True) @mock.patch('time.sleep', return_value=None) @mock.patch('os_cloud_config.keystone.initialize', autospec=True) @@ -100,7 +105,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_deploy_postconfig, mock_create_parameters_env, mock_breakpoints_cleanupm, - mock_events): + mock_events, mock_tarball): arglist = ['--templates', '--ceph-storage-scale', '3'] verifylist = [ @@ -119,6 +124,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_event = mock.Mock() mock_event.id = '1234' mock_events.return_value = [mock_events] + workflow_client = clients.workflow_engine + workflow_client.environments.get.return_value = mock.MagicMock( + variables={'environments': []}) + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') mock_check_hypervisor_stats.return_value = { 'count': 4, @@ -187,28 +197,21 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): 'StackAction': 'UPDATE', } - def _custom_create_params_env(parameters): + testcase = self + + def _custom_create_params_env(self, parameters): for key, value in six.iteritems(parameters): - self.assertEqual(value, expected_parameters[key]) + testcase.assertEqual(value, expected_parameters[key]) parameter_defaults = {"parameter_defaults": parameters} - with open(self.parameter_defaults_env_file, 'w') as temp_file: + with open(testcase.parameter_defaults_env_file, 'w') as temp_file: temp_file.write(json.dumps(parameter_defaults)) - return [self.parameter_defaults_env_file] + return [testcase.parameter_defaults_env_file] mock_create_parameters_env.side_effect = _custom_create_params_env self.cmd.take_action(parsed_args) - args, kwargs = orchestration_client.stacks.update.call_args - - self.assertEqual(args, (orchestration_client.stacks.get().id, )) - - self.assertEqual(kwargs['files'], {}) - self.assertEqual(kwargs['template'], 'template') - self.assertEqual(kwargs['environment'], mock_env) - self.assertEqual(kwargs['stack_name'], 'overcloud') - self.assertEqual(kwargs['clear_parameters'], - mock_env['parameter_defaults'].keys()) + self.assertFalse(orchestration_client.stacks.update.called) mock_get_templte_contents.assert_called_with( '/usr/share/openstack-tripleo-heat-templates/' + @@ -218,15 +221,22 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_process_multiple_env.assert_called_with( [self.parameter_defaults_env_file]) - @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env') + @mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True) + @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' + '_deploy_postconfig', autospec=True) + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) + @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env', + autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_validate_args') @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_create_parameters_env') + '_create_parameters_env', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) - @mock.patch('tripleoclient.utils.generate_overcloud_passwords') - @mock.patch('tripleoclient.utils.create_overcloudrc') + @mock.patch('tripleoclient.utils.generate_overcloud_passwords', + autospec=True) + @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True) @mock.patch('os_cloud_config.utils.clients.get_nova_bm_client', autospec=True) @mock.patch('os_cloud_config.utils.clients.get_keystone_client', @@ -262,7 +272,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_generate_overcloud_passwords, mock_create_tempest_deployer_input, mock_create_parameters_env, mock_validate_args, - mock_breakpoints_cleanup): + mock_breakpoints_cleanup, mock_tarball, + mock_postconfig, mock_get_overcloud_endpoint): arglist = ['--templates', '--ceph-storage-scale', '3'] verifylist = [ @@ -278,7 +289,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): clients = self.app.client_manager orchestration_client = clients.orchestration mock_stack = fakes.create_tht_stack() - orchestration_client.stacks.get.return_value = None + orchestration_client.stacks.get.side_effect = [None, mock.Mock()] + workflow_client = clients.workflow_engine + workflow_client.environments.get.return_value = mock.MagicMock( + variables={'environments': []}) + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') def _orch_clt_create(**kwargs): orchestration_client.stacks.get.return_value = mock_stack @@ -354,26 +370,21 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): 'StackAction': 'CREATE', } - def _custom_create_params_env(parameters): + testcase = self + + def _custom_create_params_env(self, parameters): for key, value in six.iteritems(parameters): - self.assertEqual(value, expected_parameters[key]) + testcase.assertEqual(value, expected_parameters[key]) parameter_defaults = {"parameter_defaults": parameters} - with open(self.parameter_defaults_env_file, 'w') as temp_file: + with open(testcase.parameter_defaults_env_file, 'w') as temp_file: temp_file.write(json.dumps(parameter_defaults)) - return [self.parameter_defaults_env_file] + return [testcase.parameter_defaults_env_file] mock_create_parameters_env.side_effect = _custom_create_params_env self.cmd.take_action(parsed_args) - args, kwargs = orchestration_client.stacks.create.call_args - - self.assertEqual(kwargs['files'], {}) - self.assertEqual(kwargs['template'], 'template') - self.assertEqual(kwargs['environment'], mock_env) - self.assertEqual(kwargs['stack_name'], 'overcloud') - self.assertEqual(kwargs['clear_parameters'], - mock_env['parameter_defaults'].keys()) + self.assertFalse(orchestration_client.stacks.create.called) mock_get_templte_contents.assert_called_with( '/usr/share/openstack-tripleo-heat-templates/' + @@ -381,25 +392,27 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_tempest_deployer_input.assert_called_with() mock_process_multiple_env.assert_called_with( - ['/usr/share/openstack-tripleo-heat-templates/overcloud-resource-' - 'registry-puppet.yaml', '/fake/path', - self.parameter_defaults_env_file]) + ['/fake/path', self.parameter_defaults_env_file]) mock_validate_args.assert_called_once_with(parsed_args) - mock_remove_known_hosts.assert_called_once_with('0.0.0.0') - mock_get_keystone_client.assert_called_once_with('admin', 'password', - 'admin', - 'http://0.0.0.0:8000') + mock_tarball.create_tarball.assert_called_with( + '/usr/share/openstack-tripleo-heat-templates/', mock.ANY) + mock_tarball.tarball_extract_to_swift_container.assert_called_with( + clients.tripleoclient.object_store, mock.ANY, 'overcloud') - @mock.patch("heatclient.common.event_utils.get_events") - @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env') + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) + @mock.patch("heatclient.common.event_utils.get_events", autospec=True) + @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env', + autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_deploy_postconfig') + '_deploy_postconfig', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) - @mock.patch('tripleoclient.utils.generate_overcloud_passwords') - @mock.patch('tripleoclient.utils.create_overcloudrc') + @mock.patch('tripleoclient.utils.generate_overcloud_passwords', + autospec=True) + @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True) @mock.patch('os_cloud_config.keystone.setup_endpoints', autospec=True) @mock.patch('time.sleep', return_value=None) @mock.patch('os_cloud_config.keystone.initialize', autospec=True) @@ -431,7 +444,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_tempest_deployer_input, mock_deploy_postconfig, mock_breakpoints_cleanup, - mock_events): + mock_events, mock_tarball): arglist = ['--templates', '/home/stack/tripleo-heat-templates'] verifylist = [ @@ -465,22 +478,19 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): baremetal = clients.baremetal baremetal.node.list.return_value = range(10) + workflow_client = clients.workflow_engine + workflow_client.environments.get.return_value = mock.MagicMock( + variables={'environments': []}) + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') + with mock.patch('tempfile.mkstemp') as mkstemp: mkstemp.return_value = (os.open(self.parameter_defaults_env_file, os.O_RDWR), self.parameter_defaults_env_file) self.cmd.take_action(parsed_args) - args, kwargs = orchestration_client.stacks.update.call_args - - self.assertEqual(args, (orchestration_client.stacks.get().id, )) - - self.assertEqual(kwargs['files'], {}) - self.assertEqual(kwargs['template'], 'template') - self.assertEqual(kwargs['environment'], mock_env) - self.assertEqual(kwargs['stack_name'], 'overcloud') - self.assertEqual(kwargs['clear_parameters'], - mock_env['parameter_defaults'].keys()) + self.assertFalse(orchestration_client.stacks.update.called) mock_get_templte_contents.assert_called_with( '/home/stack/tripleo-heat-templates/' + @@ -513,6 +523,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): parsed_args) self.assertFalse(mock_deploy_tht.called) + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True) @@ -527,13 +539,19 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): def test_environment_dirs(self, mock_deploy_heat, mock_update_parameters, mock_post_config, mock_utils_check_nodes, mock_utils_endpoint, - mock_utils_createrc, mock_utils_tempest): + mock_utils_createrc, mock_utils_tempest, + mock_tarball): + + clients = self.app.client_manager + workflow_client = clients.workflow_engine + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') mock_update_parameters.return_value = {} mock_utils_endpoint.return_value = 'foo.bar' tmp_dir = self.useFixture(fixtures.TempDir()) - test_env = os.path.join(tmp_dir.path, 'foo.yaml') + test_env = os.path.join(tmp_dir.path, 'foo1.yaml') env_dirs = [os.path.join(os.environ.get('HOME', ''), '.tripleo', 'environments'), tmp_dir.path] @@ -548,7 +566,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): ] def _fake_heat_deploy(self, stack, stack_name, template_path, - parameters, environments, timeout): + parameters, environments, timeout, tht_root): assert test_env in environments mock_deploy_heat.side_effect = _fake_heat_deploy @@ -556,6 +574,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): parsed_args = self.check_parser(self.cmd, arglist, verifylist) self.cmd.take_action(parsed_args) + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True) @@ -570,15 +590,21 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): def test_environment_dirs_env(self, mock_deploy_heat, mock_update_parameters, mock_post_config, mock_utils_check_nodes, mock_utils_endpoint, - mock_utils_createrc, mock_utils_tempest): + mock_utils_createrc, mock_utils_tempest, + mock_tarball): + + clients = self.app.client_manager + workflow_client = clients.workflow_engine + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') mock_update_parameters.return_value = {} mock_utils_endpoint.return_value = 'foo.bar' - tmp_dir = tempfile.NamedTemporaryFile(mode='w', delete=False).name - os.unlink(tmp_dir) - os.mkdir(tmp_dir) - test_env = os.path.join(tmp_dir, 'foo.yaml') + tmp_dir = tempfile.mkdtemp() + test_env = os.path.join(tmp_dir, 'foo2.yaml') + self.addCleanup(shutil.rmtree, tmp_dir) + with open(test_env, 'w') as temp_file: temp_file.write('#just a comment') @@ -589,15 +615,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): os.environ['TRIPLEO_ENVIRONMENT_DIRECTORY'] = tmp_dir def _fake_heat_deploy(self, stack, stack_name, template_path, - parameters, environments, timeout): + parameters, environments, timeout, tht_root): assert test_env in environments mock_deploy_heat.side_effect = _fake_heat_deploy parsed_args = self.check_parser(self.cmd, arglist, verifylist) self.cmd.take_action(parsed_args) - os.unlink(test_env) - os.rmdir(tmp_dir) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) @@ -634,14 +658,18 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_tempest_deployer_input.assert_called_with() - @mock.patch("heatclient.common.event_utils.get_events") - @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env') + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) + @mock.patch("heatclient.common.event_utils.get_events", autospec=True) + @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env', + autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' - '_deploy_postconfig') + '_deploy_postconfig', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', autospec=True) - @mock.patch('tripleoclient.utils.generate_overcloud_passwords') - @mock.patch('tripleoclient.utils.create_overcloudrc') + @mock.patch('tripleoclient.utils.generate_overcloud_passwords', + autospec=True) + @mock.patch('tripleoclient.utils.create_overcloudrc', autospec=True) @mock.patch('os_cloud_config.keystone.setup_endpoints', autospec=True) @mock.patch('time.sleep', return_value=None) @mock.patch('os_cloud_config.keystone.initialize', autospec=True) @@ -673,7 +701,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_tempest_deployer_input, mock_deploy_postconfig, mock_breakpoints_cleanup, - mock_events): + mock_events, mock_tarball): arglist = ['--templates', '--rhel-reg', '--reg-sat-url', 'https://example.com', @@ -697,6 +725,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): orchestration_client = clients.orchestration orchestration_client.stacks.get.return_value = fakes.create_tht_stack() mock_events.return_value = [] + workflow_client = clients.workflow_engine + workflow_client.environments.get.return_value = mock.MagicMock( + variables={'environments': []}) + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') mock_check_hypervisor_stats.return_value = { 'count': 4, @@ -851,7 +884,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_heat_deploy_func.assert_called_once_with( self.cmd, {}, 'overcloud', '/fake/path/' + constants.OVERCLOUD_YAML_NAMES[0], {}, - ['~/overcloud-env.json'], 1) + ['~/overcloud-env.json'], 1, '/fake/path') @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_heat_deploy') @@ -866,10 +899,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_heat_deploy_func.assert_has_calls( [mock.call({}, 'overcloud', '/fake/path/' + constants.OVERCLOUD_YAML_NAMES[0], {}, - ['~/overcloud-env.json'], 1), + ['~/overcloud-env.json'], 1, '/fake/path'), mock.call({}, 'overcloud', '/fake/path/' + constants.OVERCLOUD_YAML_NAMES[1], {}, - ['~/overcloud-env.json'], 1)]) + ['~/overcloud-env.json'], 1, '/fake/path')]) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_heat_deploy', autospec=True) @@ -924,6 +957,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.assertFalse(mock_create_ocrc.called) self.assertFalse(mock_create_tempest_deployer_input.called) + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) @mock.patch('tripleoclient.utils.check_nodes_count', autospec=True) @mock.patch('tripleoclient.utils.create_tempest_deployer_input', @@ -940,8 +975,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_oc_endpoint, mock_create_ocrc, mock_create_tempest_deployer_input, - mock_check_nodes_count): + mock_check_nodes_count, mock_tarball): clients = self.app.client_manager + + workflow_client = clients.workflow_engine + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') + network_client = clients.network network_client.stacks.get.return_value = None net = network_client.api.find_attr('networks', 'ctlplane') @@ -951,7 +991,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): with open('/tmp/environment.yaml', "w+t") as environmentfile: yaml.dump( {'templates': '/dev/null', - 'environments': ['/tmp/foo.yaml'] + 'environments': ['/tmp/foo3.yaml'] }, answerfile ) @@ -978,9 +1018,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): call_args = mock_heat_deploy.call_args[0] self.assertEqual(call_args[3], '/dev/null/overcloud-without-mergepy.yaml') - self.assertIn('/tmp/foo.yaml', call_args[5]) + self.assertIn('/tmp/foo3.yaml', call_args[5]) self.assertIn('/tmp/environment.yaml', call_args[5]) - foo_index = call_args[5].index('/tmp/foo.yaml') + foo_index = call_args[5].index('/tmp/foo3.yaml') env_index = call_args[5].index('/tmp/environment.yaml') self.assertGreater(env_index, foo_index) @@ -1071,6 +1111,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): stack, parameters, parsed_args) self.assertEqual(1, self.cmd.predeploy_errors) + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_create_parameters_env') @mock.patch('tripleoclient.utils.generate_overcloud_passwords') @@ -1088,7 +1130,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_process_multiple_env, mock_create_overcloudrc, mock_generate_overcloud_passwords, - mock_create_parameters_env): + mock_create_parameters_env, + mock_tarball): arglist = ['--templates', '--control-scale', '3'] verifylist = [ @@ -1098,6 +1141,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_get_key.return_value = "PASSWORD" + clients = self.app.client_manager + workflow_client = clients.workflow_engine + workflow_client.environments.get.return_value = mock.MagicMock( + variables={'environments': []}) + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') + mock_generate_overcloud_passwords.return_value = self._get_passwords() mock_create_env.return_value = "/fake/path" @@ -1111,6 +1161,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.cmd.take_action, parsed_args) + @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' + '_deploy_postconfig', autospec=True) + @mock.patch('tripleoclient.workflows.plan_management.tarball', + autospec=True) @mock.patch('tripleo_common.update.add_breakpoints_cleanup_into_env') @mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.' '_validate_args') @@ -1160,7 +1214,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): mock_create_tempest_deployer_input, mock_create_parameters_env, mock_validate_args, - mock_breakpoints_cleanup): + mock_breakpoints_cleanup, + mock_tarball, + mock_deploy_post_config): arglist = ['--templates', '--ceph-storage-scale', '3', '--control-scale', '3'] @@ -1178,13 +1234,22 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): clients = self.app.client_manager orchestration_client = clients.orchestration mock_stack = fakes.create_tht_stack() - orchestration_client.stacks.get.return_value = None + orchestration_client.stacks.get.side_effect = [ + None, + mock.MagicMock() + ] def _orch_clt_create(**kwargs): orchestration_client.stacks.get.return_value = mock_stack orchestration_client.stacks.create.side_effect = _orch_clt_create + workflow_client = clients.workflow_engine + workflow_client.environments.get.return_value = mock.MagicMock( + variables={'environments': []}) + workflow_client.action_executions.create.return_value = mock.MagicMock( + output='{"result":[]}') + mock_check_hypervisor_stats.return_value = { 'count': 4, 'memory_mb': 4096, @@ -1266,28 +1331,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud): self.cmd.take_action(parsed_args) - args, kwargs = orchestration_client.stacks.create.call_args - - self.assertEqual(kwargs['files'], {}) - self.assertEqual(kwargs['template'], 'template') - self.assertEqual(kwargs['environment'], mock_env) - self.assertEqual(kwargs['stack_name'], 'overcloud') - self.assertEqual(kwargs['clear_parameters'], - mock_env['parameter_defaults'].keys()) - mock_get_templte_contents.assert_called_with( '/usr/share/openstack-tripleo-heat-templates/' + constants.OVERCLOUD_YAML_NAMES[0]) mock_create_tempest_deployer_input.assert_called_with() mock_process_multiple_env.assert_called_with( - ['/usr/share/openstack-tripleo-heat-templates/overcloud-resource-' - 'registry-puppet.yaml', '/fake/path', - self.parameter_defaults_env_file]) + ['/fake/path', self.parameter_defaults_env_file]) mock_validate_args.assert_called_once_with(parsed_args) - - mock_remove_known_hosts.assert_called_once_with('0.0.0.0') - mock_get_keystone_client.assert_called_once_with('admin', 'password', - 'admin', - 'http://0.0.0.0:8000') diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index e863ea85a..0f71759a3 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -16,9 +16,11 @@ from __future__ import print_function import argparse import glob +import hashlib import json import logging import os +import os.path import re import six import tempfile @@ -41,6 +43,9 @@ from tripleo_common import update from tripleoclient import constants from tripleoclient import exceptions from tripleoclient import utils +from tripleoclient.workflows import deployment +from tripleoclient.workflows import parameters +from tripleoclient.workflows import plan_management class DeployOvercloud(command.Command): @@ -228,10 +233,13 @@ class DeployOvercloud(command.Command): return [parameter_defaults_env_file] def _heat_deploy(self, stack, stack_name, template_path, parameters, - environments, timeout): + environments, timeout, tht_root): """Verify the Baremetal nodes are available and do a stack update""" - self.log.debug("Processing environment files") + clients = self.app.client_manager + workflow_client = clients.workflow_engine + + self.log.debug("Processing environment files %s" % environments) env_files, env = ( template_utils.process_multiple_environments_and_files( environments)) @@ -253,47 +261,42 @@ class DeployOvercloud(command.Command): '(with HA).') clients = self.app.client_manager + objectclient = clients.tripleoclient.object_store + + moved_files = self._upload_missing_files( + stack_name, objectclient, files, tht_root) + self._process_and_upload_environment( + stack_name, objectclient, env, moved_files, tht_root, + workflow_client) + + self._start_mistral_deploy(clients, stack, stack_name) + + def _start_mistral_deploy(self, clients, stack, plan_name): + + deployment.deploy(clients, container=plan_name, + queue_name=str(uuid.uuid4())) + orchestration_client = clients.orchestration - self.log.debug("Deploying stack: %s", stack_name) - self.log.debug("Deploying template: %s", template) - self.log.debug("Deploying parameters: %s", parameters) - self.log.debug("Deploying environment: %s", env) - self.log.debug("Deploying files: %s", files) - - stack_args = { - 'stack_name': stack_name, - 'template': template, - 'environment': env, - 'files': files, - 'clear_parameters': env['parameter_defaults'].keys(), - } - - if timeout: - stack_args['timeout_mins'] = timeout - if stack is None: self.log.info("Performing Heat stack create") action = 'CREATE' marker = None - orchestration_client.stacks.create(**stack_args) else: self.log.info("Performing Heat stack update") # Make sure existing parameters for stack are reused - stack_args['existing'] = 'true' # Find the last top-level event to use for the first marker events = event_utils.get_events(orchestration_client, - stack_id=stack_name, + stack_id=plan_name, event_args={'sort_dir': 'desc', 'limit': 1}) marker = events[0].id if events else None action = 'UPDATE' - orchestration_client.stacks.update(stack.id, **stack_args) - + time.sleep(10) verbose_events = self.app_args.verbose_level > 0 create_result = utils.wait_for_stack_ready( - orchestration_client, stack_name, marker, action, verbose_events) + orchestration_client, plan_name, marker, action, verbose_events) if not create_result: if stack is None: raise exceptions.DeploymentError("Heat Stack create failed.") @@ -314,16 +317,107 @@ class DeployOvercloud(command.Command): environments.append(f) return environments + def _process_and_upload_environment(self, container_name, swift_client, + env, moved_files, tht_root, mistral): + """Process the environment and upload to Swift + + The environment at this point should be the result of the merged + custom user environments. We need to look at the paths in the + environment and update any that changed when they were uploaded to + swift. + """ + + file_prefix = "file://" + + if 'resource_registry' in env: + for name, path in env['resource_registry'].items(): + if not isinstance(path, six.string_types): + continue + if path in moved_files: + new_path = moved_files[path] + env['resource_registry'][name] = new_path + elif path.startswith(file_prefix): + path = path[len(file_prefix):] + if path.startswith(tht_root): + path = path[len(tht_root):] + # We want to make sure all the paths are relative. + if path.startswith("/"): + path = path[1:] + env['resource_registry'][name] = path + + # Parameters are removed from the environment and sent to the update + # parameters action, this stores them in the Mistral environment and + # means the UI can find them. + if 'parameter_defaults' in env: + params = env.pop('parameter_defaults') + parameters.update_parameters(mistral, container=container_name, + parameters=params) + + contents = yaml.safe_dump(env) + + swift_path = "user-environment.yaml" + swift_client.put_object(container_name, swift_path, contents) + + mistral_env = mistral.environments.get(container_name) + mistral_env.variables['environments'].append({'path': swift_path}) + mistral.environments.update( + name=container_name, + variables=mistral_env.variables + ) + + def _upload_missing_files(self, container_name, swift_client, files_dict, + tht_root): + """Find the files referenced in custom environments and upload them + + Heat environments can be passed to be included in the deployment, these + files can include references to other files anywhere on the local + file system. These need to be discovered and uploaded to Swift. When + they have been uploaded to Swift the path to them will be different, + the new paths are store din the file_relocation dict, which is returned + and used by _process_and_upload_environment which will merge the + environment and update paths to the relative Swift path. + """ + + file_relocation = {} + file_prefix = "file://" + + for fullpath, contents in files_dict.items(): + + if not fullpath.startswith(file_prefix): + continue + + path = fullpath[len(file_prefix):] + + if path.startswith(tht_root): + # This should already be uploaded. + continue + + filename = os.path.basename(path) + checksum = hashlib.md5() + checksum.update(filename) + digest = checksum.hexdigest() + swift_path = "user-files/{}-{}".format(digest, filename) + swift_client.put_object(container_name, swift_path, contents) + file_relocation[fullpath] = swift_path + + return file_relocation + def _deploy_tripleo_heat_templates(self, stack, parsed_args): """Deploy the fixed templates in TripleO Heat Templates""" clients = self.app.client_manager network_client = clients.network + workflow_client = clients.workflow_engine parameters = self._update_parameters( parsed_args, network_client, stack) tht_root = parsed_args.templates + plans = plan_management.list_deployment_plans(workflow_client) + if parsed_args.stack not in plans: + plan_management.create_plan_from_templates( + clients, parsed_args.stack, tht_root) + print("Deploying templates in the directory {0}".format( os.path.abspath(tht_root))) @@ -333,13 +427,11 @@ class DeployOvercloud(command.Command): # parameters and keystone cert is generated on create only env_path = utils.create_environment_file() environments = [] - add_registry = False if stack is None: self.log.debug("Creating Keystone certificates") keystone_pki.generate_certs_into_json(env_path, False) environments.append(env_path) - add_registry = True if parsed_args.environment_directories: environments.extend(self._load_environment_directories( @@ -349,19 +441,8 @@ class DeployOvercloud(command.Command): if parsed_args.rhel_reg: reg_env = self._create_registration_env(parsed_args) environments.extend(reg_env) - add_registry = True if parsed_args.environment_files: environments.extend(parsed_args.environment_files) - add_registry = True - - if add_registry: - # default resource registry file should be passed only - # when creating a new stack, or when custom environments are - # specified, otherwise it might overwrite - # resource_registries in existing stack - resource_registry_path = os.path.join( - tht_root, constants.RESOURCE_REGISTRY_NAME) - environments.insert(0, resource_registry_path) self._try_overcloud_deploy_with_compat_yaml( tht_root, stack, parsed_args.stack, parameters, environments, @@ -375,7 +456,7 @@ class DeployOvercloud(command.Command): overcloud_yaml = os.path.join(tht_root, overcloud_yaml_name) try: self._heat_deploy(stack, stack_name, overcloud_yaml, - parameters, environments, timeout) + parameters, environments, timeout, tht_root) except six.moves.urllib.error.URLError as e: messages.append(str(e.reason)) else: diff --git a/tripleoclient/workflows/deployment.py b/tripleoclient/workflows/deployment.py new file mode 100644 index 000000000..63466589f --- /dev/null +++ b/tripleoclient/workflows/deployment.py @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import print_function + +import pprint + + +def deploy(clients, **workflow_input): + + workflow_client = clients.workflow_engine + tripleoclients = clients.tripleoclient + queue_name = workflow_input['queue_name'] + + execution = workflow_client.executions.create( + 'tripleo.deployment.v1.deploy_plan', + workflow_input=workflow_input + ) + + with tripleoclients.messaging_websocket(queue_name) as ws: + message = ws.wait_for_message(execution.id) + assert message['status'] == "SUCCESS", pprint.pformat(message) diff --git a/tripleoclient/workflows/parameters.py b/tripleoclient/workflows/parameters.py new file mode 100644 index 000000000..8fa131a75 --- /dev/null +++ b/tripleoclient/workflows/parameters.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from tripleoclient.workflows import base + + +def update_parameters(workflow_client, **input_): + return base.call_action(workflow_client, 'tripleo.update_parameters', + **input_)