Copy user provided files into working tht root

Implements a standard pattern to normalize user provided templates by
copying them into the working tht path.  When a user provides a -e for a
file that already exists within the templates path, it is updated to the
expected working tht root.  For files that are outside of the templates
path, they will be copied into the root of the working tht root and
processed using the process-templates.py during the deployment. This
allows for a standard relative path and supports j2 files.

Note that this means you will not be able to provide two identically
named files that live outside of the templates folder since they will
conflict. An explicit exception is thrown if the system identifies an
already existing file.

Change-Id: I2138a5dcf6a2aff04f7e522cbdf2d5ce94283dc1
Related-Blueprint: all-in-one
This commit is contained in:
Alex Schultz 2018-05-31 17:15:37 -06:00
parent 33e16d6f73
commit edf976ee31
4 changed files with 152 additions and 46 deletions

View File

@ -308,6 +308,61 @@ class TestDeployUndercloud(TestPluginV1):
mock_yaml_dump.assert_has_calls([mock.call(rewritten_env,
default_flow_style=False)])
@mock.patch('shutil.copy')
@mock.patch('os.path.exists', return_value=False)
def test_normalize_user_templates(self, mock_exists, mock_copy):
user_tht_root = '/userroot'
tht_root = '/thtroot'
env_files = [
'/home/basic.yaml',
'/home/dir/dir.yaml',
'home/relative.yaml',
'file.yaml',
'~/tilde.yaml',
'../../../dots.yaml',
'/userroot/template.yaml',
'/userroot/tht/tht.yaml',
]
expected = [
'/thtroot/basic.yaml',
'/thtroot/dir.yaml',
'/thtroot/relative.yaml',
'/thtroot/file.yaml',
'/thtroot/tilde.yaml',
'/thtroot/dots.yaml',
'/thtroot/template.yaml',
'/thtroot/tht/tht.yaml'
]
results = self.cmd._normalize_user_templates(user_tht_root,
tht_root,
env_files)
self.assertEqual(expected, results)
self.assertEqual(mock_copy.call_count, 6)
@mock.patch('os.path.exists', return_value=True)
def test_normalize_user_templates_exists(self, mock_exists):
user_tht_root = '/userroot'
tht_root = '/thtroot'
env_files = ['/home/basic.yaml']
self.assertRaises(exceptions.DeploymentError,
self.cmd._normalize_user_templates,
user_tht_root,
tht_root,
env_files)
@mock.patch('os.path.exists', return_value=True)
def test_normalize_user_templates_trailing_slash(self, mock_exists):
user_tht_root = '/userroot/'
tht_root = '/thtroot'
env_files = ['/userroot/basic.yaml']
expected = ['/thtroot/basic.yaml']
results = self.cmd._normalize_user_templates(user_tht_root,
tht_root,
env_files)
self.assertEqual(expected, results)
@mock.patch('os.path.exists', return_value=False)
@mock.patch('tripleoclient.v1.tripleo_deploy.Deploy.'
'_create_working_dirs')
@mock.patch('heatclient.common.template_utils.'
@ -334,8 +389,11 @@ class TestDeployUndercloud(TestPluginV1):
mock_process_multiple_environments,
mock_hc_get_templ_cont,
mock_hc_process,
mock_createdirs):
mock_createdirs,
mock_exists):
mock_update_pass_env.return_value = '/my/tripleo-heat-installer-' \
'templates/passwords.yaml'
mock_run.return_value = 0
# logic handled in _standalone_deploy and _create_working_dirs
@ -360,19 +418,20 @@ class TestDeployUndercloud(TestPluginV1):
expected_env = [
'/my/tripleo-heat-installer-templates/'
'overcloud-resource-registry-puppet.yaml',
mock.ANY,
'/my/tripleo-heat-installer-templates/passwords.yaml',
'/my/tripleo-heat-installer-templates/'
'environments/config-download-environment.yaml',
'/my/tripleo-heat-installer-templates/'
'environments/deployed-server-noop-ctlplane.yaml',
'/tmp/thtroot/puppet/foo.yaml',
'/tmp/thtroot//docker/bar.yaml',
'/tmp/thtroot42/notouch.yaml',
'~/custom.yaml',
'something.yaml',
'../../../outside.yaml',
'/my/tripleo-heat-installer-templates/'
'tripleoclient-hosts-portmaps.yaml', 'foo.yaml']
'tripleoclient-hosts-portmaps.yaml',
'/my/tripleo-heat-installer-templates/puppet/foo.yaml',
'/my/tripleo-heat-installer-templates//docker/bar.yaml',
'/my/tripleo-heat-installer-templates/notouch.yaml',
'/my/tripleo-heat-installer-templates/custom.yaml',
'/my/tripleo-heat-installer-templates/something.yaml',
'/my/tripleo-heat-installer-templates/outside.yaml',
'foo.yaml']
environment = self.cmd._setup_heat_environments(parsed_args)

View File

@ -112,7 +112,8 @@ class TestUndercloudInstall(TestPluginV1):
'services/undercloud-haproxy.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml', '--output-dir=/foo',
'--cleanup', '-e', '/foo/tripleo-heat-installer-templates/'
'--cleanup', '-e',
'/foo/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=/tmp/install-undercloud.log'])
@ -271,7 +272,8 @@ class TestUndercloudInstall(TestPluginV1):
'services/undercloud-haproxy.yaml', '-e',
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml', '--output-dir=/home/stack',
'--cleanup', '-e', '/home/stack/tripleo-heat-installer-templates/'
'--cleanup', '-e',
'/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=/tmp/install-undercloud.log'])
@ -329,7 +331,7 @@ class TestUndercloudInstall(TestPluginV1):
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml',
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-heat-installer-templates/'
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--debug', '--log-file=/tmp/install-undercloud.log'])
@ -389,7 +391,7 @@ class TestUndercloudInstall(TestPluginV1):
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml',
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-heat-installer-templates/'
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=/tmp/install-undercloud.log'])
@ -481,7 +483,7 @@ class TestUndercloudUpgrade(TestPluginV1):
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml',
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-heat-installer-templates/'
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=/tmp/install-undercloud.log'])
@ -538,7 +540,7 @@ class TestUndercloudUpgrade(TestPluginV1):
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml',
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-heat-installer-templates/'
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--log-file=/tmp/install-undercloud.log'])
@ -599,6 +601,6 @@ class TestUndercloudUpgrade(TestPluginV1):
'/usr/share/openstack-tripleo-heat-templates/environments/'
'services/undercloud-keepalived.yaml',
'--output-dir=/home/stack', '--cleanup',
'-e', '/home/stack/tripleo-heat-installer-templates/'
'-e', '/home/stack/tripleo-config-generated-env-files/'
'undercloud_parameters.yaml',
'--debug', '--log-file=/tmp/install-undercloud.log'])

View File

@ -397,6 +397,54 @@ class Deploy(command.Command):
return orchestration_client
def _normalize_user_templates(self, user_tht_root, tht_root, env_files=[]):
"""copy environment files into tht render path
This assumes any env file that includes user_tht_root has already
been copied into tht_root.
:param user_tht_root: string path to the user's template dir
:param tht_root: string path to our deployed tht_root
:param env_files: list of paths to environment files
:return list of absolute pathed environment files that exist in
tht_root
"""
environments = []
# normalize the user template path to ensure it doesn't have a trailing
# slash
user_tht = os.path.abspath(user_tht_root)
for env_path in env_files:
self.log.debug("Processing file %s" % env_path)
abs_env_path = os.path.abspath(env_path)
if (abs_env_path.startswith(user_tht_root) and
((user_tht + '/') in env_path or
(user_tht + '/') in abs_env_path or
user_tht == abs_env_path or
user_tht == env_path)):
# file is in tht and will be copied, so just update path
new_env_path = env_path.replace(user_tht + '/',
tht_root + '/')
self.log.debug("Redirecting %s to %s"
% (abs_env_path, new_env_path))
environments.append(new_env_path)
elif abs_env_path.startswith(tht_root):
self.log.debug("File already in tht_root %s")
environments.append(abs_env_path)
else:
self.log.debug("File outside of tht_root %s, copying in")
# file is outside of THT, just copy it in
# TODO(aschultz): probably shouldn't be flattened?
target_dest = os.path.join(tht_root,
os.path.basename(abs_env_path))
if os.path.exists(target_dest):
raise exceptions.DeploymentError("%s already exists, "
"please rename the "
"file to something else"
% target_dest)
shutil.copy(abs_env_path, tht_root)
environments.append(target_dest)
return environments
def _setup_heat_environments(self, parsed_args):
"""Process tripleo heat templates with jinja and deploy into work dir
@ -407,15 +455,18 @@ class Deploy(command.Command):
overcloud-resource-registry-puppet.yaml and passwords files.
"""
# TODO(aschultz): This probably needs to get thought about because
# we likely need to do this for any environment file that gets passed
# in the deploy. This file breaks the seperation between standalone
# and undercloud deployment so we need to not hardcode
# undercloud_parameters.yaml
shutil.copy(os.path.join(os.path.abspath(parsed_args.output_dir),
'tripleo-config-generated-env-files',
'undercloud_parameters.yaml'),
self.tht_render)
self.log.warning(_("** Handling template files **"))
env_files = []
# TODO(aschultz): in overcloud deploy we have a --environments-dir
# we might want to handle something similar for this
if parsed_args.environment_files:
env_files.extend(parsed_args.environment_files)
# ensure any user provided templates get copied into tht_render
environments = self._normalize_user_templates(parsed_args.templates,
self.tht_render,
env_files)
# generate jinja templates by its work dir location
self.log.debug(_("Using roles file %s") % self.roles_file)
@ -429,12 +480,9 @@ class Deploy(command.Command):
self.log.error(msg)
raise exceptions.DeploymentError(msg)
self.log.info(_("Deploying templates in the directory {0}").format(
os.path.abspath(self.tht_render)))
self.log.warning(_("** Creating Environment file **"))
environments = []
# NOTE(aschultz): the next set of environment files are system included
# so we have to include them at the front of our environment list so a
# user can override anything in them.
resource_registry_path = os.path.join(
self.tht_render, 'overcloud-resource-registry-puppet.yaml')
environments.insert(0, resource_registry_path)
@ -447,16 +495,16 @@ class Deploy(command.Command):
deployed_server_env = os.path.join(
self.tht_render, 'environments',
'config-download-environment.yaml')
environments.append(deployed_server_env)
environments.insert(2, deployed_server_env)
# use deployed-server because we run os-collect-config locally
deployed_server_env = os.path.join(
self.tht_render, 'environments',
'deployed-server-noop-ctlplane.yaml')
environments.append(deployed_server_env)
environments.insert(3, deployed_server_env)
if parsed_args.environment_files:
environments.extend(parsed_args.environment_files)
self.log.info(_("Deploying templates in the directory {0}").format(
os.path.abspath(self.tht_render)))
maps_file = os.path.join(self.tht_render,
'tripleoclient-hosts-portmaps.yaml')
@ -482,8 +530,10 @@ class Deploy(command.Command):
with open(maps_file, 'w') as env_file:
yaml.safe_dump({'parameter_defaults': tmp_env}, env_file,
default_flow_style=False)
environments.append(maps_file)
environments.insert(4, maps_file)
# NOTE(aschultz): this doesn't get copied into tht_root but
# we always include the hieradata override stuff last.
if parsed_args.hieradata_override:
environments.append(self._process_hieradata_overrides(
parsed_args.hieradata_override,

View File

@ -524,22 +524,17 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=False,
env_data['UndercloudNetConfigOverride'] = net_config_json
# NOTE(bogdando): the generated env files are stored another path than
# picked up later - from the working dir containing renedered
# heat templates. We can not store files by that path right here, as that
# directory gets removed to be populated with the entire templates
# directory tree later. And the generated files should be copied in only
# after that.
# NOTE(bogdando): the generated env files are stored another path then
# picked up later.
# NOTE(aschultz): We copy this into the tht root that we save because
# we move any user provided environment files into this root later.
tempdir = os.path.join(os.path.abspath(CONF['output_dir']),
'tripleo-config-generated-env-files')
params_file = os.path.join(tempdir, 'undercloud_parameters.yaml')
if not os.path.isdir(tempdir):
os.mkdir(tempdir)
utils.write_env_file(env_data, params_file, registry_overwrites)
params_file_real = os.path.join(os.path.abspath(CONF['output_dir']),
'tripleo-heat-installer-templates',
'undercloud_parameters.yaml')
deploy_args += ['-e', params_file_real]
deploy_args += ['-e', params_file]
if CONF.get('hieradata_override', None):
data_file = CONF['hieradata_override']