Merge "Allow user environments override generated ones"

This commit is contained in:
Zuul 2021-05-07 09:46:34 +00:00 committed by Gerrit Code Review
commit 0503b7e36f
2 changed files with 130 additions and 188 deletions

View File

@ -231,7 +231,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.build_stack_data', autospec=True)
@mock.patch('tripleo_common.utils.plan.default_image_params',
autospec=True)
return_value={})
@mock.patch('tripleoclient.utils.get_rc_params', autospec=True)
@mock.patch('tripleo_common.utils.plan.generate_passwords',
return_value={})
@ -510,7 +510,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_copy.assert_called_once()
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'deploy_tripleo_heat_templates', autospec=True)
def test_jinja2_env_path(self, mock_deploy_tht, mock_create_env):
@ -530,8 +530,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.check_service_vips_migrated_to_service')
@mock.patch('tripleoclient.utils.get_rc_params', autospec=True)
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', autospec=True)
@mock.patch('tripleoclient.utils.process_multiple_environments',
autospec=True)
@mock.patch('tripleoclient.utils.check_nic_config_with_ansible')
@mock.patch('tripleoclient.utils.copy_clouds_yaml')
@mock.patch('tripleoclient.utils.check_stack_network_matches_env_files')
@ -542,7 +542,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_update_parameters', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_heat_deploy', autospec=True)
def test_environment_dirs(self, mock_deploy_heat, mock_create_env,
@ -588,7 +588,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
self.assertEqual(*args)
def _fake_heat_deploy(self, stack, stack_name, template_path,
parameters, environments, timeout, tht_root,
environments, timeout, tht_root,
env, run_validations,
roles_file,
env_files_tracker=None,
@ -603,7 +603,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_rc_params.return_value = {'password': 'password',
'region': 'region1'}
mock_deploy_heat.side_effect = _fake_heat_deploy
mock_create_env.return_value = ({}, [])
mock_create_env.return_value = []
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
mock_copy.assert_called_once()
@ -672,13 +672,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
def test_try_overcloud_deploy_with_first_template_existing(
self, mock_heat_deploy_func):
result = self.cmd._try_overcloud_deploy_with_compat_yaml(
'/fake/path', {}, 'overcloud', {}, ['~/overcloud-env.json'], 1,
'/fake/path', {}, 'overcloud', ['~/overcloud-env.json'], 1,
{}, False, None, None)
# If it returns None it succeeded
self.assertIsNone(result)
mock_heat_deploy_func.assert_called_once_with(
self.cmd, {}, 'overcloud',
'/fake/path/' + constants.OVERCLOUD_YAML_NAME, {},
'/fake/path/' + constants.OVERCLOUD_YAML_NAME,
['~/overcloud-env.json'], 1, '/fake/path', {}, False,
None, deployment_options=None, env_files_tracker=None)
@ -689,7 +689,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_heat_deploy_func.side_effect = Exception('error')
self.assertRaises(ValueError,
self.cmd._try_overcloud_deploy_with_compat_yaml,
'/fake/path', mock.ANY, mock.ANY, mock.ANY,
'/fake/path', mock.ANY, mock.ANY,
mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY,
None)
@ -701,7 +701,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
Exception('/fake/path not found')
try:
self.cmd._try_overcloud_deploy_with_compat_yaml(
'/fake/path', mock.ANY, mock.ANY, mock.ANY,
'/fake/path', mock.ANY, mock.ANY,
mock.ANY, mock.ANY, mock.ANY, mock.ANY, mock.ANY,
None)
except ValueError as value_error:
@ -711,7 +711,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_get_undercloud_host_entry', autospec=True,
return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'deploy_tripleo_heat_templates', autospec=True)
def test_dry_run(self, mock_deploy, mock_create_env,
@ -828,78 +828,15 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
self.assertEqual(call_args[3],
self.tmp_dir.join(
'tripleo-heat-templates/overcloud.yaml'))
self.assertEqual(call_args[7],
self.assertEqual(call_args[6],
self.tmp_dir.join('tripleo-heat-templates'))
self.assertIn('Test', call_args[8]['resource_registry'])
self.assertIn('Test2', call_args[8]['resource_registry'])
self.assertIn('Test', call_args[7]['resource_registry'])
self.assertIn('Test2', call_args[7]['resource_registry'])
utils_fixture.mock_deploy_tht.assert_called_with(
output_dir=self.cmd.working_dir)
mock_copy.assert_called_once()
@mock.patch('tripleoclient.utils.check_service_vips_migrated_to_service')
@mock.patch('tripleoclient.utils.get_rc_params', autospec=True)
@mock.patch('tripleo_common.utils.plan.generate_passwords',
return_value={})
@mock.patch(
'tripleo_common.image.kolla_builder.container_images_prepare_multi',
return_value={})
@mock.patch('tripleoclient.workflows.roles.get_roles_data',
autospec=True, return_value={})
@mock.patch('tripleoclient.utils.check_nic_config_with_ansible')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_get_undercloud_host_entry', autospec=True,
return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane')
@mock.patch('tripleoclient.utils.check_stack_network_matches_env_files')
@mock.patch('tripleoclient.utils.check_ceph_fsid_matches_env_files')
@mock.patch('tripleoclient.utils.check_swift_and_rgw')
@mock.patch('tripleoclient.utils.create_parameters_env', autospec=True)
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', autospec=True)
@mock.patch('heatclient.common.template_utils.get_template_contents',
autospec=True)
def test_ntp_server_mandatory(self, mock_get_template_contents,
mock_process_env,
mock_create_parameters_env,
mock_stack_network_check,
mock_ceph_fsid, mock_swift_rgw,
mock_get_undercloud_host_entry,
mock_nic_ansible,
mock_roles_data,
mock_image_prepare,
mock_generate_password,
mock_rc_params,
mock_check_service_vip_migr):
clients = self.app.client_manager
orchestration_client = clients.orchestration
mock_stack = fakes.create_tht_stack()
orchestration_client.stacks.get.return_value = mock_stack
arglist = ['--templates']
verifylist = [
('templates', '/usr/share/openstack-tripleo-heat-templates/'),
]
def _custom_create_params_env(parameters, tht_root,
stack):
parameters.update({"ControllerCount": 3})
parameter_defaults = {"parameter_defaults": parameters}
return parameter_defaults
mock_create_parameters_env.side_effect = _custom_create_params_env
mock_env = fakes.create_env()
mock_process_env.return_value = [{}, mock_env]
mock_get_template_contents.return_value = [{}, "template"]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
mock_rc_params.return_value = {'password': 'password',
'region': 'region1'}
self.assertRaises(exceptions.InvalidConfiguration,
self.cmd.take_action,
parsed_args)
@mock.patch('tripleoclient.utils.build_stack_data', autospec=True)
@mock.patch('tripleo_common.utils.plan.default_image_params',
autospec=True)
@ -1035,7 +972,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_get_undercloud_host_entry', autospec=True,
return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'deploy_tripleo_heat_templates', autospec=True)
def test_deployed_server(self, mock_deploy, mock_create_env,
@ -1074,7 +1011,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_get_undercloud_host_entry', autospec=True,
return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'deploy_tripleo_heat_templates', autospec=True)
def test_config_download(
@ -1115,7 +1052,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_get_undercloud_host_entry', autospec=True,
return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'deploy_tripleo_heat_templates', autospec=True)
def test_config_download_setup_only(
@ -1151,7 +1088,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_copy.assert_called_once()
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.utils.get_rc_params', autospec=True)
@mock.patch('tripleoclient.utils.copy_clouds_yaml')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
@ -1193,7 +1130,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_copy.assert_called_once()
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.utils.get_rc_params', autospec=True)
@mock.patch('tripleoclient.utils.copy_clouds_yaml')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
@ -1246,7 +1183,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_get_undercloud_host_entry', autospec=True,
return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'deploy_tripleo_heat_templates', autospec=True)
def test_override_ansible_cfg(
@ -1278,7 +1215,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_copy.assert_called_once()
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.utils.check_service_vips_migrated_to_service')
@mock.patch('tripleo_common.utils.plan.default_image_params',
autospec=True)
@ -1290,8 +1227,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
return_value={})
@mock.patch('tripleoclient.workflows.roles.get_roles_data',
autospec=True, return_value={})
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', autospec=True)
@mock.patch('tripleoclient.utils.process_multiple_environments',
autospec=True)
@mock.patch('tripleoclient.utils.check_nic_config_with_ansible')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_get_ctlplane_attrs', autospec=True, return_value={})
@ -1322,7 +1259,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
clients = self.app.client_manager
orchestration_client = clients.orchestration
orchestration_client.stacks.get.return_value = fakes.create_tht_stack()
mock_create_parameters_env.return_value = ({}, [])
mock_create_parameters_env.return_value = []
arglist = ['--templates', '--overcloud-ssh-port-timeout', '42',
'--timeout', '451']
@ -1342,8 +1279,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
self.cmd.take_action(parsed_args)
self.assertIn(
[mock.call(mock.ANY, mock.ANY, 'overcloud', mock.ANY,
{'StackAction': 'UPDATE'}, mock.ANY,
451, mock.ANY, mock.ANY, False, None,
mock.ANY, 451, mock.ANY, mock.ANY, False, None,
deployment_options={}, env_files_tracker=mock.ANY)],
mock_hd.mock_calls)
self.assertIn(
@ -1370,7 +1306,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_get_undercloud_host_entry', autospec=True,
return_value='192.168.0.1 uc.ctlplane.localhost uc.ctlplane')
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'create_params_and_env_files', autospec=True)
'create_env_files', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'deploy_tripleo_heat_templates', autospec=True)
def test_config_download_only_timeout(

View File

@ -199,7 +199,7 @@ class DeployOvercloud(command.Command):
'be ignored because --limit has been specified.')
self.log.warning(msg)
def _heat_deploy(self, stack, stack_name, template_path, parameters,
def _heat_deploy(self, stack, stack_name, template_path,
env_files, timeout, tht_root, env,
run_validations,
roles_file,
@ -247,57 +247,43 @@ class DeployOvercloud(command.Command):
new_tht_root)
return new_tht_root, tht_root
def create_params_and_env_files(self, new_tht_root, user_tht_root,
parsed_args):
def build_image_params(self, env_files, parsed_args,
new_tht_root, user_tht_root):
image_params = plan_utils.default_image_params()
if not parsed_args.disable_container_prepare:
if parsed_args.environment_directories:
env_files.extend(utils.load_environment_directories(
parsed_args.environment_directories))
if parsed_args.environment_files:
env_files.extend(parsed_args.environment_files)
_, env = utils.process_multiple_environments(
env_files, new_tht_root, user_tht_root,
cleanup=(not parsed_args.no_cleanup))
updated_params = kolla_builder.container_images_prepare_multi(
env, roles.get_roles_data(parsed_args.roles_file,
new_tht_root), dry_run=True)
if updated_params:
image_params.update(updated_params)
return image_params
def create_env_files(self, stack, parsed_args,
new_tht_root, user_tht_root):
self.log.debug("Creating Environment files")
created_env_files = []
created_env_files.append(
os.path.join(new_tht_root, constants.DEFAULT_RESOURCE_REGISTRY))
if parsed_args.environment_directories:
created_env_files.extend(utils.load_environment_directories(
parsed_args.environment_directories))
if parsed_args.deployed_server:
created_env_files.append(
os.path.join(
new_tht_root,
constants.DEPLOYED_SERVER_ENVIRONMENT))
if parsed_args.environment_files:
created_env_files.extend(parsed_args.environment_files)
created_env_files.extend(
self._provision_baremetal(parsed_args, new_tht_root))
_, env = utils.process_multiple_environments(
created_env_files, new_tht_root, user_tht_root,
cleanup=(not parsed_args.no_cleanup))
default_image_params = plan_utils.default_image_params()
image_params = {}
if not parsed_args.disable_container_prepare:
image_params = kolla_builder.container_images_prepare_multi(
env, roles.get_roles_data(parsed_args.roles_file,
new_tht_root), dry_run=True)
parameters = {}
if image_params:
default_image_params.update(image_params)
parameters.update(default_image_params)
parameters = self.build_image_params(
created_env_files, parsed_args, new_tht_root, user_tht_root)
self._update_parameters(
parsed_args, parameters, new_tht_root, user_tht_root)
return parameters, created_env_files
def deploy_tripleo_heat_templates(self, stack, parsed_args,
new_tht_root, user_tht_root,
parameters, created_env_files):
"""Deploy the fixed templates in TripleO Heat Templates"""
self.log.info("Processing templates in the directory {0}".format(
os.path.abspath(new_tht_root)))
stack_is_new = stack is None
parameters['StackAction'] = 'CREATE' if stack_is_new else 'UPDATE'
@ -305,16 +291,42 @@ class DeployOvercloud(command.Command):
parameters, new_tht_root, parsed_args.stack)
created_env_files.extend(param_env)
deployment_options = {}
if parsed_args.deployment_python_interpreter:
deployment_options['ansible_python_interpreter'] = \
parsed_args.deployment_python_interpreter
if stack:
env_path = utils.create_breakpoint_cleanup_env(
new_tht_root, parsed_args.stack)
created_env_files.extend(env_path)
if parsed_args.deployed_server:
created_env_files.append(
os.path.join(
new_tht_root,
constants.DEPLOYED_SERVER_ENVIRONMENT))
created_env_files.extend(
self._provision_baremetal(parsed_args, new_tht_root))
if parsed_args.environment_directories:
created_env_files.extend(utils.load_environment_directories(
parsed_args.environment_directories))
if parsed_args.environment_files:
created_env_files.extend(parsed_args.environment_files)
return created_env_files
def deploy_tripleo_heat_templates(self, stack, parsed_args,
new_tht_root, user_tht_root,
created_env_files):
"""Deploy the fixed templates in TripleO Heat Templates"""
self.log.info("Processing templates in the directory {0}".format(
os.path.abspath(new_tht_root)))
deployment_options = {}
if parsed_args.deployment_python_interpreter:
deployment_options['ansible_python_interpreter'] = \
parsed_args.deployment_python_interpreter
self.log.debug("Processing environment files %s" % created_env_files)
env_files_tracker = []
env_files, env = utils.process_multiple_environments(
@ -369,19 +381,13 @@ class DeployOvercloud(command.Command):
# check migration to service vips managed by servce
utils.check_service_vips_migrated_to_service(stack, env)
# FIXME(shardy) It'd be better to validate this via mistral
# e.g part of the plan create/update workflow
number_controllers = int(parameters.get('ControllerCount', 0))
if number_controllers > 1:
if not env.get('parameter_defaults').get('NtpServer'):
raise exceptions.InvalidConfiguration(
'Specify --ntp-server as parameter or NtpServer in '
'environments when using multiple controllers '
'(with HA).')
if parsed_args.heat_type != 'installed':
self.setup_ephemeral_heat(
parsed_args, env.get('parameter_defaults'))
self._try_overcloud_deploy_with_compat_yaml(
new_tht_root, stack,
parsed_args.stack, parameters, env_files,
parsed_args.stack, env_files,
parsed_args.timeout, env,
parsed_args.run_validations,
parsed_args.roles_file,
@ -409,7 +415,7 @@ class DeployOvercloud(command.Command):
utils.safe_write(reloc_path, files_dict[fullpath])
def _try_overcloud_deploy_with_compat_yaml(self, tht_root, stack,
stack_name, parameters,
stack_name,
env_files, timeout,
env, run_validations,
roles_file,
@ -418,7 +424,7 @@ class DeployOvercloud(command.Command):
overcloud_yaml = os.path.join(tht_root, constants.OVERCLOUD_YAML_NAME)
try:
self._heat_deploy(stack, stack_name, overcloud_yaml,
parameters, env_files, timeout,
env_files, timeout,
tht_root, env,
run_validations,
roles_file,
@ -590,6 +596,27 @@ class DeployOvercloud(command.Command):
}
)
def setup_ephemeral_heat(self, parsed_args, parameters):
self.log.info("Using tripleo-deploy with "
"ephemeral heat-all for stack operation")
api_container_image = parameters['ContainerHeatApiImage']
engine_container_image = \
parameters['ContainerHeatEngineImage']
restore_db = (parsed_args.setup_only or
parsed_args.config_download_only)
self.heat_launcher = utils.get_heat_launcher(
parsed_args.heat_type,
api_container_image=api_container_image,
engine_container_image=engine_container_image,
heat_dir=os.path.join(self.working_dir,
'heat-launcher'),
use_tmp_dir=False,
rm_heat=parsed_args.rm_heat,
skip_heat_pull=parsed_args.skip_heat_pull)
self.orchestration_client = \
utils.launch_heat(self.heat_launcher, restore_db=restore_db)
self.clients.orchestration = self.orchestration_client
def get_parser(self, prog_name):
# add_help doesn't work properly, set it to False:
parser = argparse.ArgumentParser(
@ -1004,50 +1031,29 @@ class DeployOvercloud(command.Command):
stack_create = None
start = time.time()
if parsed_args.heat_type == 'installed':
stack = utils.get_stack(self.orchestration_client,
parsed_args.stack)
stack_create = stack is None
if stack_create:
self.log.info("No stack found, "
"will be doing a stack create")
else:
self.log.info("Stack found, "
"will be doing a stack update")
new_tht_root, user_tht_root = \
self.create_template_dirs(parsed_args)
parameters, created_env_files = \
self.create_params_and_env_files(
new_tht_root, user_tht_root, parsed_args)
if parsed_args.heat_type != 'installed':
self.log.info("Using tripleo-deploy with "
"ephemeral heat-all for stack operation")
api_container_image = parameters['ContainerHeatApiImage']
engine_container_image = \
parameters['ContainerHeatEngineImage']
restore_db = (parsed_args.setup_only or
parsed_args.config_download_only)
self.heat_launcher = utils.get_heat_launcher(
parsed_args.heat_type,
api_container_image=api_container_image,
engine_container_image=engine_container_image,
heat_dir=os.path.join(self.working_dir,
'heat-launcher'),
use_tmp_dir=False,
rm_heat=parsed_args.rm_heat,
skip_heat_pull=parsed_args.skip_heat_pull)
self.orchestration_client = \
utils.launch_heat(self.heat_launcher, restore_db=restore_db)
self.clients.orchestration = self.orchestration_client
created_env_files = self.create_env_files(
stack, parsed_args, new_tht_root, user_tht_root)
try:
if parsed_args.heat_type == 'installed':
stack = utils.get_stack(self.orchestration_client,
parsed_args.stack)
stack_create = stack is None
if stack_create:
self.log.info("No stack found, "
"will be doing a stack create")
else:
self.log.info("Stack found, "
"will be doing a stack update")
if not (parsed_args.config_download_only or
parsed_args.setup_only):
self.deploy_tripleo_heat_templates(
stack, parsed_args, new_tht_root, user_tht_root,
parameters, created_env_files)
stack, parsed_args, new_tht_root,
user_tht_root, created_env_files)
except Exception:
if parsed_args.heat_type != 'installed' and self.heat_launcher:
self.log.info("Stopping ephemeral heat.")