openstack
/
python-tripleoclient
Code Issues Proposed changes

Remove unused services in tripleo deploy 

When rendering templates, check for services mapped to OS::Heat::None
and rewrite roles data to remove them. It reduces the numbers of
resources created, and improve in particular container image building.

Change-Id: I37aa673937681d5142feb74e89ca85fdb471db57
This commit is contained in:
Thomas Herve 2018-12-10 14:32:13 +01:00
parent 51a3e57ed7
commit bf6f2b4083
2 changed files with 146 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
tripleoclient/tests/v1/tripleo/test_tripleo_deploy.py
View File

@ -126,19 +126,27 @@ class TestDeployUndercloud(TestPluginV1):
mock_data.return_value = [
{'name': 'Bar'}, {'name': 'Foo', 'tags': ['primary']}
]
self.assertEqual(self.cmd._get_primary_role_name(parsed_args), 'Foo')
self.assertEqual(
self.cmd._get_primary_role_name(parsed_args.roles_file,
parsed_args.templates),
'Foo')
@mock.patch('tripleoclient.utils.fetch_roles_file', return_value=None)
def test_get_primary_role_name_none_defined(self, mock_data):
parsed_args = self.check_parser(self.cmd, [], [])
self.assertEqual(self.cmd._get_primary_role_name(parsed_args),
'Controller')
self.assertEqual(
self.cmd._get_primary_role_name(parsed_args.roles_file,
parsed_args.templates),
'Controller')
@mock.patch('tripleoclient.utils.fetch_roles_file')
def test_get_primary_role_name_no_primary(self, mock_data):
parsed_args = mock.Mock()
mock_data.return_value = [{'name': 'Bar'}, {'name': 'Foo'}]
self.assertEqual(self.cmd._get_primary_role_name(parsed_args), 'Bar')
self.assertEqual(
self.cmd._get_primary_role_name(parsed_args.roles_file,
parsed_args.templates),
'Bar')
@mock.patch('os.path.exists', side_effect=[True, False])
@mock.patch('shutil.copytree')
@ -399,6 +407,92 @@ class TestDeployUndercloud(TestPluginV1):
mock_yaml_dump.assert_has_calls([mock.call(rewritten_env,
default_flow_style=False)])
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_populate_templates_dir')
@mock.patch('tripleoclient.utils.fetch_roles_file')
@mock.patch('tripleoclient.utils.rel_or_abs_path')
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'get_template_contents', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.environment_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('heatclient.common.template_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_setup_heat_environments', autospec=True)
@mock.patch('yaml.safe_dump', autospec=True)
@mock.patch('yaml.safe_load', autospec=True)
@mock.patch('six.moves.builtins.open')
@mock.patch('tempfile.NamedTemporaryFile', autospec=True)
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare_multi')
def test_deploy_tripleo_heat_templates_remove(self,
mock_cipm,
mock_temp, mock_open,
mock_yaml_load,
mock_yaml_dump,
mock_setup_heat_envs,
mock_hc_templ_parse,
mock_hc_env_parse,
mock_hc_get_templ_cont,
mock_hc_process,
mock_norm_path,
mock_fetch_roles,
mock_populate):
def hc_process(*args, **kwargs):
if 'myenv.yaml' in kwargs['env_path']:
env = {
'resource_registry': {
'OS::TripleO::Services::Foo': 'OS::Heat::None'}}
return ({}, env)
else:
return ({}, {})
mock_fetch_roles.return_value = [
{'name': 'Bar', 'ServicesDefault': [
'OS::TripleO::Services::Foo', 'OS::TripleO::Services::Bar']},
{'name': 'Foo', 'tags': ['primary']}
]
def set_tht(templates):
self.cmd.tht_render = "tht_from"
mock_populate.side_effect = set_tht
mock_cipm.return_value = {}
mock_hc_process.side_effect = hc_process
parsed_args = self.check_parser(self.cmd,
['--templates', '/tmp/thtroot'], [])
rewritten_role = [
{'name': 'Bar', 'ServicesDefault': ['OS::TripleO::Services::Bar']},
{'name': 'Foo', 'tags': ['primary']}
]
myenv = {'resource_registry': {
'OS::Foo::Bar': '../outside.yaml',
'OS::Foo::Baz': './inside.yaml',
'OS::Foo::Qux': '/tmp/thtroot/abs.yaml',
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
'OS::Foo::Corge': '/tmp/thtroot/puppet/foo.yaml'
}
}
mock_yaml_load.return_value = myenv
mock_setup_heat_envs.return_value = [
'./inside.yaml', '/tmp/thtroot/abs.yaml',
'/tmp/thtroot/puppet/foo.yaml',
'/tmp/thtroot/environments/myenv.yaml',
'../outside.yaml']
self.cmd._deploy_tripleo_heat_templates(self.orc, parsed_args)
mock_yaml_dump.assert_has_calls([mock.call(rewritten_role)])
@mock.patch('shutil.copy')
@mock.patch('os.path.exists', return_value=False)
def test_normalize_user_templates(self, mock_exists, mock_copy):
@ -481,7 +575,8 @@ class TestDeployUndercloud(TestPluginV1):
self.cmd.output_dir = 'tht_to'
self.cmd.tht_render = 'tht_from'
self.cmd.stack_action = 'UPDATE'
environment = self.cmd._setup_heat_environments(parsed_args)
environment = self.cmd._setup_heat_environments(parsed_args.roles_file,
parsed_args)
self.assertIn(dropin, environment)
mock_open.assert_has_calls([mock.call(dropin, 'w')])
@ -650,7 +745,8 @@ class TestDeployUndercloud(TestPluginV1):
with mock.patch('os.path.abspath', side_effect=abs_path_stub):
with mock.patch('os.path.isfile'):
environment = self.cmd._setup_heat_environments(parsed_args)
environment = self.cmd._setup_heat_environments(
parsed_args.roles_file, parsed_args)
self.assertEqual(expected_env, environment)
@ -694,16 +790,13 @@ class TestDeployUndercloud(TestPluginV1):
self.ansible_playbook_cmd, '-i', '/tmp/inventory.yaml',
'deploy_steps_playbook.yaml'])
@mock.patch('tripleoclient.utils.fetch_roles_file')
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare_multi')
def test_prepare_container_images(self, mock_cipm, rolesdata_mock):
parsed_args = mock.Mock()
def test_prepare_container_images(self, mock_cipm):
env = {'parameter_defaults': {}}
mock_cipm.return_value = {'FooImage': 'foo/bar:baz'}
rolesdata_mock.return_value = [{'name': 'Compute'}]
self.cmd._prepare_container_images(env, parsed_args)
self.cmd._prepare_container_images(env, [{'name': 'Compute'}])
mock_cipm.assert_called_once_with(
env,

56
tripleoclient/v1/tripleo_deploy.py
View File

@ -171,10 +171,10 @@ class Deploy(command.Command):
plan_env = parsed_args.plan_environment_file
return plan_env
def _get_primary_role_name(self, parsed_args):
def _get_primary_role_name(self, roles_file_path, templates):
"""Return the primary role name"""
roles_data = utils.fetch_roles_file(
self._get_roles_file_path(parsed_args), parsed_args.templates)
roles_file_path, templates)
if not roles_data:
# TODO(aschultz): should this be Undercloud instead?
return 'Controller'
@ -233,7 +233,7 @@ class Deploy(command.Command):
self.tht_render = os.path.join(self.output_dir,
'tripleo-heat-installer-templates')
# Clear dir since we're using a static name and shutils.copytree
# needs the fodler to not exist. We'll generate the
# needs the folder to not exist. We'll generate the
# contents each time. This should clear the folder on the first
# run of this function.
shutil.rmtree(self.tht_render, ignore_errors=True)
@ -582,7 +582,7 @@ class Deploy(command.Command):
environments.append(target_dest)
return environments
def _setup_heat_environments(self, parsed_args):
def _setup_heat_environments(self, roles_file_path, parsed_args):
"""Process tripleo heat templates with jinja and deploy into work dir
* Process j2/install additional templates there
@ -607,13 +607,11 @@ class Deploy(command.Command):
parsed_args.templates, self.tht_render, env_files)
# generate jinja templates by its work dir location
self.log.debug(_("Using roles "
"file %s") % self._get_roles_file_path(parsed_args))
self.log.debug(_("Using roles file %s") % roles_file_path)
process_templates = os.path.join(parsed_args.templates,
'tools/process-templates.py')
args = [self.python_cmd, process_templates, '--roles-data',
self._get_roles_file_path(parsed_args), '--output-dir',
self.tht_render]
roles_file_path, '--output-dir', self.tht_render]
if utils.run_command_and_log(self.log, args, cwd=self.tht_render) != 0:
# TODO(aschultz): improve error messaging
msg = _("Problems generating templates.")
@ -675,7 +673,8 @@ class Deploy(command.Command):
tmp_env.update(self._generate_portmap_parameters(
ip, ip_nw, c_ip, p_ip,
stack_name=parsed_args.stack,
role_name=self._get_primary_role_name(parsed_args)))
role_name=self._get_primary_role_name(
roles_file_path, parsed_args.templates)))
with open(maps_file, 'w') as env_file:
yaml.safe_dump({'parameter_defaults': tmp_env}, env_file,
@ -704,9 +703,7 @@ class Deploy(command.Command):
return environments + user_environments
def _prepare_container_images(self, env, parsed_args):
roles_data = utils.fetch_roles_file(
self._get_roles_file_path(parsed_args), parsed_args.templates)
def _prepare_container_images(self, env, roles_data):
image_params = kolla_builder.container_images_prepare_multi(
env, roles_data, dry_run=True)
@ -720,9 +717,11 @@ class Deploy(command.Command):
def _deploy_tripleo_heat_templates(self, orchestration_client,
parsed_args):
"""Deploy the fixed templates in TripleO Heat Templates"""
roles_file_path = self._get_roles_file_path(parsed_args)
# sets self.tht_render to the working dir with deployed templates
environments = self._setup_heat_environments(parsed_args)
environments = self._setup_heat_environments(
roles_file_path, parsed_args)
# rewrite paths to consume t-h-t env files from the working dir
self.log.debug(_("Processing environment files %s") % environments)
@ -730,7 +729,36 @@ class Deploy(command.Command):
environments, self.tht_render, parsed_args.templates,
cleanup=parsed_args.cleanup)
self._prepare_container_images(env, parsed_args)
roles_data = utils.fetch_roles_file(
roles_file_path, parsed_args.templates)
to_remove = set()
for key, value in env.get('resource_registry', {}).items():
if (key.startswith('OS::TripleO::Services::') and
value == 'OS::Heat::None'):
to_remove.add(key)
if to_remove:
for role in roles_data:
for service in to_remove:
try:
role.get('ServicesDefault', []).remove(service)
except ValueError:
pass
self.log.info('Removing unused services, updating roles')
# This will clean up the directory and set it up again
self.tht_render = None
self._populate_templates_dir(parsed_args.templates)
roles_file_path = os.path.join(
self.tht_render, 'roles-data-override.yaml')
with open(roles_file_path, "w") as f:
f.write(yaml.safe_dump(roles_data))
# Redo the dance
environments = self._setup_heat_environments(
roles_file_path, parsed_args)
env_files, env = utils.process_multiple_environments(
environments, self.tht_render, parsed_args.templates,
cleanup=parsed_args.cleanup)
self._prepare_container_images(env, roles_data)
self.log.debug(_("Getting template contents"))
template_path = os.path.join(self.tht_render, 'overcloud.yaml')