Copy user provided files into working tht root
Implements a standard pattern to normalize user provided templates by copying them into the working tht path. When a user provides a -e for a file that already exists within the templates path, it is updated to the expected working tht root. For files that are outside of the templates path, they will be copied into the root of the working tht root and processed using the process-templates.py during the deployment. This allows for a standard relative path and supports j2 files. Note that this means you will not be able to provide two identically named files that live outside of the templates folder since they will conflict. An explicit exception is thrown if the system identifies an already existing file. Change-Id: I2138a5dcf6a2aff04f7e522cbdf2d5ce94283dc1 Related-Blueprint: all-in-one
This commit is contained in:
parent
33e16d6f73
commit
edf976ee31
|
@ -308,6 +308,61 @@ class TestDeployUndercloud(TestPluginV1):
|
|||
mock_yaml_dump.assert_has_calls([mock.call(rewritten_env,
|
||||
default_flow_style=False)])
|
||||
|
||||
@mock.patch('shutil.copy')
|
||||
@mock.patch('os.path.exists', return_value=False)
|
||||
def test_normalize_user_templates(self, mock_exists, mock_copy):
|
||||
user_tht_root = '/userroot'
|
||||
tht_root = '/thtroot'
|
||||
env_files = [
|
||||
'/home/basic.yaml',
|
||||
'/home/dir/dir.yaml',
|
||||
'home/relative.yaml',
|
||||
'file.yaml',
|
||||
'~/tilde.yaml',
|
||||
'../../../dots.yaml',
|
||||
'/userroot/template.yaml',
|
||||
'/userroot/tht/tht.yaml',
|
||||
]
|
||||
expected = [
|
||||
'/thtroot/basic.yaml',
|
||||
'/thtroot/dir.yaml',
|
||||
'/thtroot/relative.yaml',
|
||||
'/thtroot/file.yaml',
|
||||
'/thtroot/tilde.yaml',
|
||||
'/thtroot/dots.yaml',
|
||||
'/thtroot/template.yaml',
|
||||
'/thtroot/tht/tht.yaml'
|
||||
]
|
||||
results = self.cmd._normalize_user_templates(user_tht_root,
|
||||
tht_root,
|
||||
env_files)
|
||||
|
||||
self.assertEqual(expected, results)
|
||||
self.assertEqual(mock_copy.call_count, 6)
|
||||
|
||||
@mock.patch('os.path.exists', return_value=True)
|
||||
def test_normalize_user_templates_exists(self, mock_exists):
|
||||
user_tht_root = '/userroot'
|
||||
tht_root = '/thtroot'
|
||||
env_files = ['/home/basic.yaml']
|
||||
self.assertRaises(exceptions.DeploymentError,
|
||||
self.cmd._normalize_user_templates,
|
||||
user_tht_root,
|
||||
tht_root,
|
||||
env_files)
|
||||
|
||||
@mock.patch('os.path.exists', return_value=True)
|
||||
def test_normalize_user_templates_trailing_slash(self, mock_exists):
|
||||
user_tht_root = '/userroot/'
|
||||
tht_root = '/thtroot'
|
||||
env_files = ['/userroot/basic.yaml']
|
||||
expected = ['/thtroot/basic.yaml']
|
||||
results = self.cmd._normalize_user_templates(user_tht_root,
|
||||
tht_root,
|
||||
env_files)
|
||||
self.assertEqual(expected, results)
|
||||
|
||||
@mock.patch('os.path.exists', return_value=False)
|
||||
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
|
||||
'_create_working_dirs')
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
|
@ -334,8 +389,11 @@ class TestDeployUndercloud(TestPluginV1):
|
|||
mock_process_multiple_environments,
|
||||
mock_hc_get_templ_cont,
|
||||
mock_hc_process,
|
||||
mock_createdirs):
|
||||
mock_createdirs,
|
||||
mock_exists):
|
||||
|
||||
mock_update_pass_env.return_value = '/my/tripleo-heat-installer-' \
|
||||
'templates/passwords.yaml'
|
||||
mock_run.return_value = 0
|
||||
|
||||
# logic handled in _standalone_deploy and _create_working_dirs
|
||||
|
@ -360,19 +418,20 @@ class TestDeployUndercloud(TestPluginV1):
|
|||
expected_env = [
|
||||
'/my/tripleo-heat-installer-templates/'
|
||||
'overcloud-resource-registry-puppet.yaml',
|
||||
mock.ANY,
|
||||
'/my/tripleo-heat-installer-templates/passwords.yaml',
|
||||
'/my/tripleo-heat-installer-templates/'
|
||||
'environments/config-download-environment.yaml',
|
||||
'/my/tripleo-heat-installer-templates/'
|
||||
'environments/deployed-server-noop-ctlplane.yaml',
|
||||
'/tmp/thtroot/puppet/foo.yaml',
|
||||
'/tmp/thtroot//docker/bar.yaml',
|
||||
'/tmp/thtroot42/notouch.yaml',
|
||||
'~/custom.yaml',
|
||||
'something.yaml',
|
||||
'../../../outside.yaml',
|
||||
'/my/tripleo-heat-installer-templates/'
|
||||
'tripleoclient-hosts-portmaps.yaml', 'foo.yaml']
|
||||
'tripleoclient-hosts-portmaps.yaml',
|
||||
'/my/tripleo-heat-installer-templates/puppet/foo.yaml',
|
||||
'/my/tripleo-heat-installer-templates//docker/bar.yaml',
|
||||
'/my/tripleo-heat-installer-templates/notouch.yaml',
|
||||
'/my/tripleo-heat-installer-templates/custom.yaml',
|
||||
'/my/tripleo-heat-installer-templates/something.yaml',
|
||||
'/my/tripleo-heat-installer-templates/outside.yaml',
|
||||
'foo.yaml']
|
||||
|
||||
environment = self.cmd._setup_heat_environments(parsed_args)
|
||||
|
||||
|
|
|
@ -112,7 +112,8 @@ class TestUndercloudInstall(TestPluginV1):
|
|||
'services/undercloud-haproxy.yaml', '-e',
|
||||
'/usr/share/openstack-tripleo-heat-templates/environments/'
|
||||
'services/undercloud-keepalived.yaml', '--output-dir=/foo',
|
||||
'--cleanup', '-e', '/foo/tripleo-heat-installer-templates/'
|
||||
'--cleanup', '-e',
|
||||
'/foo/tripleo-config-generated-env-files/'
|
||||
'undercloud_parameters.yaml',
|
||||
'--log-file=/tmp/install-undercloud.log'])
|
||||
|
||||
|
@ -271,7 +272,8 @@ class TestUndercloudInstall(TestPluginV1):
|
|||
'services/undercloud-haproxy.yaml', '-e',
|
||||
'/usr/share/openstack-tripleo-heat-templates/environments/'
|
||||
'services/undercloud-keepalived.yaml', '--output-dir=/home/stack',
|
||||
'--cleanup', '-e', '/home/stack/tripleo-heat-installer-templates/'
|
||||
'--cleanup', '-e',
|
||||
'/home/stack/tripleo-config-generated-env-files/'
|
||||
'undercloud_parameters.yaml',
|
||||
'--log-file=/tmp/install-undercloud.log'])
|
||||
|
||||
|
@ -329,7 +331,7 @@ class TestUndercloudInstall(TestPluginV1):
|
|||
'/usr/share/openstack-tripleo-heat-templates/environments/'
|
||||
'services/undercloud-keepalived.yaml',
|
||||
'--output-dir=/home/stack', '--cleanup',
|
||||
'-e', '/home/stack/tripleo-heat-installer-templates/'
|
||||
'-e', '/home/stack/tripleo-config-generated-env-files/'
|
||||
'undercloud_parameters.yaml',
|
||||
'--debug', '--log-file=/tmp/install-undercloud.log'])
|
||||
|
||||
|
@ -389,7 +391,7 @@ class TestUndercloudInstall(TestPluginV1):
|
|||
'/usr/share/openstack-tripleo-heat-templates/environments/'
|
||||
'services/undercloud-keepalived.yaml',
|
||||
'--output-dir=/home/stack', '--cleanup',
|
||||
'-e', '/home/stack/tripleo-heat-installer-templates/'
|
||||
'-e', '/home/stack/tripleo-config-generated-env-files/'
|
||||
'undercloud_parameters.yaml',
|
||||
'--log-file=/tmp/install-undercloud.log'])
|
||||
|
||||
|
@ -481,7 +483,7 @@ class TestUndercloudUpgrade(TestPluginV1):
|
|||
'/usr/share/openstack-tripleo-heat-templates/environments/'
|
||||
'services/undercloud-keepalived.yaml',
|
||||
'--output-dir=/home/stack', '--cleanup',
|
||||
'-e', '/home/stack/tripleo-heat-installer-templates/'
|
||||
'-e', '/home/stack/tripleo-config-generated-env-files/'
|
||||
'undercloud_parameters.yaml',
|
||||
'--log-file=/tmp/install-undercloud.log'])
|
||||
|
||||
|
@ -538,7 +540,7 @@ class TestUndercloudUpgrade(TestPluginV1):
|
|||
'/usr/share/openstack-tripleo-heat-templates/environments/'
|
||||
'services/undercloud-keepalived.yaml',
|
||||
'--output-dir=/home/stack', '--cleanup',
|
||||
'-e', '/home/stack/tripleo-heat-installer-templates/'
|
||||
'-e', '/home/stack/tripleo-config-generated-env-files/'
|
||||
'undercloud_parameters.yaml',
|
||||
'--log-file=/tmp/install-undercloud.log'])
|
||||
|
||||
|
@ -599,6 +601,6 @@ class TestUndercloudUpgrade(TestPluginV1):
|
|||
'/usr/share/openstack-tripleo-heat-templates/environments/'
|
||||
'services/undercloud-keepalived.yaml',
|
||||
'--output-dir=/home/stack', '--cleanup',
|
||||
'-e', '/home/stack/tripleo-heat-installer-templates/'
|
||||
'-e', '/home/stack/tripleo-config-generated-env-files/'
|
||||
'undercloud_parameters.yaml',
|
||||
'--debug', '--log-file=/tmp/install-undercloud.log'])
|
||||
|
|
|
@ -397,6 +397,54 @@ class Deploy(command.Command):
|
|||
|
||||
return orchestration_client
|
||||
|
||||
def _normalize_user_templates(self, user_tht_root, tht_root, env_files=[]):
|
||||
"""copy environment files into tht render path
|
||||
|
||||
This assumes any env file that includes user_tht_root has already
|
||||
been copied into tht_root.
|
||||
|
||||
:param user_tht_root: string path to the user's template dir
|
||||
:param tht_root: string path to our deployed tht_root
|
||||
:param env_files: list of paths to environment files
|
||||
:return list of absolute pathed environment files that exist in
|
||||
tht_root
|
||||
"""
|
||||
environments = []
|
||||
# normalize the user template path to ensure it doesn't have a trailing
|
||||
# slash
|
||||
user_tht = os.path.abspath(user_tht_root)
|
||||
for env_path in env_files:
|
||||
self.log.debug("Processing file %s" % env_path)
|
||||
abs_env_path = os.path.abspath(env_path)
|
||||
if (abs_env_path.startswith(user_tht_root) and
|
||||
((user_tht + '/') in env_path or
|
||||
(user_tht + '/') in abs_env_path or
|
||||
user_tht == abs_env_path or
|
||||
user_tht == env_path)):
|
||||
# file is in tht and will be copied, so just update path
|
||||
new_env_path = env_path.replace(user_tht + '/',
|
||||
tht_root + '/')
|
||||
self.log.debug("Redirecting %s to %s"
|
||||
% (abs_env_path, new_env_path))
|
||||
environments.append(new_env_path)
|
||||
elif abs_env_path.startswith(tht_root):
|
||||
self.log.debug("File already in tht_root %s")
|
||||
environments.append(abs_env_path)
|
||||
else:
|
||||
self.log.debug("File outside of tht_root %s, copying in")
|
||||
# file is outside of THT, just copy it in
|
||||
# TODO(aschultz): probably shouldn't be flattened?
|
||||
target_dest = os.path.join(tht_root,
|
||||
os.path.basename(abs_env_path))
|
||||
if os.path.exists(target_dest):
|
||||
raise exceptions.DeploymentError("%s already exists, "
|
||||
"please rename the "
|
||||
"file to something else"
|
||||
% target_dest)
|
||||
shutil.copy(abs_env_path, tht_root)
|
||||
environments.append(target_dest)
|
||||
return environments
|
||||
|
||||
def _setup_heat_environments(self, parsed_args):
|
||||
"""Process tripleo heat templates with jinja and deploy into work dir
|
||||
|
||||
|
@ -407,15 +455,18 @@ class Deploy(command.Command):
|
|||
overcloud-resource-registry-puppet.yaml and passwords files.
|
||||
"""
|
||||
|
||||
# TODO(aschultz): This probably needs to get thought about because
|
||||
# we likely need to do this for any environment file that gets passed
|
||||
# in the deploy. This file breaks the seperation between standalone
|
||||
# and undercloud deployment so we need to not hardcode
|
||||
# undercloud_parameters.yaml
|
||||
shutil.copy(os.path.join(os.path.abspath(parsed_args.output_dir),
|
||||
'tripleo-config-generated-env-files',
|
||||
'undercloud_parameters.yaml'),
|
||||
self.tht_render)
|
||||
self.log.warning(_("** Handling template files **"))
|
||||
env_files = []
|
||||
|
||||
# TODO(aschultz): in overcloud deploy we have a --environments-dir
|
||||
# we might want to handle something similar for this
|
||||
if parsed_args.environment_files:
|
||||
env_files.extend(parsed_args.environment_files)
|
||||
|
||||
# ensure any user provided templates get copied into tht_render
|
||||
environments = self._normalize_user_templates(parsed_args.templates,
|
||||
self.tht_render,
|
||||
env_files)
|
||||
|
||||
# generate jinja templates by its work dir location
|
||||
self.log.debug(_("Using roles file %s") % self.roles_file)
|
||||
|
@ -429,12 +480,9 @@ class Deploy(command.Command):
|
|||
self.log.error(msg)
|
||||
raise exceptions.DeploymentError(msg)
|
||||
|
||||
self.log.info(_("Deploying templates in the directory {0}").format(
|
||||
os.path.abspath(self.tht_render)))
|
||||
|
||||
self.log.warning(_("** Creating Environment file **"))
|
||||
environments = []
|
||||
|
||||
# NOTE(aschultz): the next set of environment files are system included
|
||||
# so we have to include them at the front of our environment list so a
|
||||
# user can override anything in them.
|
||||
resource_registry_path = os.path.join(
|
||||
self.tht_render, 'overcloud-resource-registry-puppet.yaml')
|
||||
environments.insert(0, resource_registry_path)
|
||||
|
@ -447,16 +495,16 @@ class Deploy(command.Command):
|
|||
deployed_server_env = os.path.join(
|
||||
self.tht_render, 'environments',
|
||||
'config-download-environment.yaml')
|
||||
environments.append(deployed_server_env)
|
||||
environments.insert(2, deployed_server_env)
|
||||
|
||||
# use deployed-server because we run os-collect-config locally
|
||||
deployed_server_env = os.path.join(
|
||||
self.tht_render, 'environments',
|
||||
'deployed-server-noop-ctlplane.yaml')
|
||||
environments.append(deployed_server_env)
|
||||
environments.insert(3, deployed_server_env)
|
||||
|
||||
if parsed_args.environment_files:
|
||||
environments.extend(parsed_args.environment_files)
|
||||
self.log.info(_("Deploying templates in the directory {0}").format(
|
||||
os.path.abspath(self.tht_render)))
|
||||
|
||||
maps_file = os.path.join(self.tht_render,
|
||||
'tripleoclient-hosts-portmaps.yaml')
|
||||
|
@ -482,8 +530,10 @@ class Deploy(command.Command):
|
|||
with open(maps_file, 'w') as env_file:
|
||||
yaml.safe_dump({'parameter_defaults': tmp_env}, env_file,
|
||||
default_flow_style=False)
|
||||
environments.append(maps_file)
|
||||
environments.insert(4, maps_file)
|
||||
|
||||
# NOTE(aschultz): this doesn't get copied into tht_root but
|
||||
# we always include the hieradata override stuff last.
|
||||
if parsed_args.hieradata_override:
|
||||
environments.append(self._process_hieradata_overrides(
|
||||
parsed_args.hieradata_override,
|
||||
|
|
|
@ -524,22 +524,17 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=False,
|
|||
|
||||
env_data['UndercloudNetConfigOverride'] = net_config_json
|
||||
|
||||
# NOTE(bogdando): the generated env files are stored another path than
|
||||
# picked up later - from the working dir containing renedered
|
||||
# heat templates. We can not store files by that path right here, as that
|
||||
# directory gets removed to be populated with the entire templates
|
||||
# directory tree later. And the generated files should be copied in only
|
||||
# after that.
|
||||
# NOTE(bogdando): the generated env files are stored another path then
|
||||
# picked up later.
|
||||
# NOTE(aschultz): We copy this into the tht root that we save because
|
||||
# we move any user provided environment files into this root later.
|
||||
tempdir = os.path.join(os.path.abspath(CONF['output_dir']),
|
||||
'tripleo-config-generated-env-files')
|
||||
params_file = os.path.join(tempdir, 'undercloud_parameters.yaml')
|
||||
if not os.path.isdir(tempdir):
|
||||
os.mkdir(tempdir)
|
||||
utils.write_env_file(env_data, params_file, registry_overwrites)
|
||||
params_file_real = os.path.join(os.path.abspath(CONF['output_dir']),
|
||||
'tripleo-heat-installer-templates',
|
||||
'undercloud_parameters.yaml')
|
||||
deploy_args += ['-e', params_file_real]
|
||||
deploy_args += ['-e', params_file]
|
||||
|
||||
if CONF.get('hieradata_override', None):
|
||||
data_file = CONF['hieradata_override']
|
||||
|
|
Loading…
Reference in New Issue