Prepare t-h-t for undercloud in a work dir

The issue here is that the undercloud heat installer
default uses template in /usr/share and renders the
result of the process-templates.py in place.

Process templates in a temporary work dir created
from the source --templates dir under the --output-dir.
Delete generated templates and temp files after the
undercloud deployment done or failed, when --cleanup
is specified.
Ensure the --output-dir exists for all of the cases.

The --output-dir and --cleanup may be passed
either as the 'undercloud deploy' CLI args, or via
the undercloud.conf file, as 'output_dir' and 'cleanup'.
The latter way only works for 'undercloud install --use-heat'.

Also, share process_multiple_environments in utils lib.
Reuse the shared process_multiple_environments for
undercloud and overcloud heat installers and add unit tests.

Additionally, fix the shared process_multiple_environments
redirect/rewrite: only redirect and rewrite paths, if fully
matched.

Closes-Bug: #1752272
Closes-Bug: #1749683
Change-Id: Ib0c2e3ffd81742441400d27857afae457d71a424
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
This commit is contained in:
Bogdan Dobrelya 2018-02-09 15:56:13 +01:00
parent f3948b5532
commit 0b3b55288b
8 changed files with 478 additions and 103 deletions

View File

@ -0,0 +1,22 @@
---
fixes:
- |
Fix undercloud heat installer renders Heat templates in
`/usr/share`, which contains t-h-t installed from the package.
features:
- |
New command line arguments `--output-dir` and `--cleanup`
define the heat templates processing rules for undercloud:
``undercloud deploy --cleanup --output-dir /tmp/tht``.
The `output_dir` and `cleanup` configuration options
for `undercloud.conf` may be used the same way and allow to
configure ``undercloud install --use-heat`` behavior.
upgrade:
- |
The default value for `--output-dir` is changed for
undercloud heat installer to `$HOME/.undercloud-heat-installer`.
The content of the processed heat templates will be persisted
under the given path as `$output_dir/$tempdir/templates`, for
each run of the undercloud deploy or install commands, unless
the `cleanup` mode is requested.

View File

@ -13,10 +13,14 @@
# under the License.
#
import os
TRIPLEO_HEAT_TEMPLATES = "/usr/share/openstack-tripleo-heat-templates/"
OVERCLOUD_YAML_NAME = "overcloud.yaml"
OVERCLOUD_ROLES_FILE = "roles_data.yaml"
UNDERCLOUD_ROLES_FILE = "roles_data_undercloud.yaml"
UNDERCLOUD_OUTPUT_DIR = os.path.join(os.environ.get('HOME'),
'.undercloud-heat-installer')
OVERCLOUD_NETWORKS_FILE = "network_data.yaml"
RHEL_REGISTRATION_EXTRACONFIG_NAME = (
"extraconfig/pre_deploy/rhel-registration/")
@ -27,7 +31,8 @@ USER_ENVIRONMENT = 'user-environment.yaml'
USER_PARAMETERS = 'user-environments/tripleoclient-parameters.yaml'
# This directory may contain additional environments to use during deploy
DEFAULT_ENV_DIRECTORY = "~/.tripleo/environments"
DEFAULT_ENV_DIRECTORY = os.path.join(os.environ.get('HOME'),
'.tripleo', 'environments')
TRIPLEO_PUPPET_MODULES = "/usr/share/openstack-puppet/modules/"
UPGRADE_CONVERGE_FILE = "major-upgrade-converge-docker.yaml"

View File

@ -19,6 +19,9 @@ import datetime
import mock
import os.path
import tempfile
from heatclient import exc as hc_exc
from uuid import uuid4
from unittest import TestCase
@ -638,3 +641,100 @@ class TestStoreCliParam(TestCase):
mock_isdir.return_value = True
mock_open.side_effect = IOError()
self.assertRaises(IOError, utils.store_cli_param, "command", self.args)
class ProcessMultipleEnvironments(TestCase):
def setUp(self):
self.tht_root = '/twd/templates'
self.user_tht_root = '/tmp/thtroot/'
self.created_env_files = [
'./inside.yaml', '/tmp/thtroot/abs.yaml',
'/tmp/thtroot/puppet/foo.yaml',
'/tmp/thtroot/environments/myenv.yaml',
'/tmp/thtroot42/notouch.yaml',
'./tmp/thtroot/notouch2.yaml',
'../outside.yaml']
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'get_template_contents', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.environment_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('heatclient.common.template_format.'
'parse', autospec=True, return_value=dict())
def test_redirect_templates_paths(self,
mock_hc_templ_parse,
mock_hc_env_parse,
mock_hc_get_templ_cont,
mock_hc_process):
utils.process_multiple_environments(self.created_env_files,
self.tht_root,
self.user_tht_root)
mock_hc_process.assert_has_calls([
mock.call(env_path='./inside.yaml'),
mock.call(env_path='/twd/templates/abs.yaml'),
mock.call(env_path='/twd/templates/puppet/foo.yaml'),
mock.call(env_path='/twd/templates/environments/myenv.yaml'),
mock.call(env_path='/tmp/thtroot42/notouch.yaml'),
mock.call(env_path='./tmp/thtroot/notouch2.yaml'),
mock.call(env_path='../outside.yaml')])
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'get_template_contents', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.environment_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('heatclient.common.template_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('yaml.safe_dump', autospec=True)
@mock.patch('yaml.safe_load', autospec=True)
@mock.patch('six.moves.builtins.open')
@mock.patch('tempfile.NamedTemporaryFile', autospec=True)
def test_rewrite_env_files(self,
mock_temp, mock_open,
mock_yaml_load,
mock_yaml_dump,
mock_hc_templ_parse,
mock_hc_env_parse,
mock_hc_get_templ_cont,
mock_hc_process):
def hc_process(*args, **kwargs):
if 'abs.yaml' in kwargs['env_path']:
raise hc_exc.CommandError
else:
return ({}, {})
mock_hc_process.side_effect = hc_process
rewritten_env = {'resource_registry': {
'OS::Foo::Bar': '/twd/outside.yaml',
'OS::Foo::Baz': '/twd/templates/inside.yaml',
'OS::Foo::Qux': '/twd/templates/abs.yaml',
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
'OS::Foo::Corge': '/twd/templates/puppet/foo.yaml'
}
}
myenv = {'resource_registry': {
'OS::Foo::Bar': '../outside.yaml',
'OS::Foo::Baz': './inside.yaml',
'OS::Foo::Qux': '/tmp/thtroot/abs.yaml',
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
'OS::Foo::Corge': '/tmp/thtroot/puppet/foo.yaml'
}
}
mock_yaml_load.return_value = myenv
utils.process_multiple_environments(self.created_env_files,
self.tht_root,
self.user_tht_root, False)
mock_yaml_dump.assert_has_calls([mock.call(rewritten_env,
default_flow_style=False)])

View File

@ -16,6 +16,8 @@
import mock
import os
from heatclient import exc as hc_exc
from tripleoclient.tests.v1.test_plugin import TestPluginV1
# Load the plugin init module for the plugin list and show commands
@ -28,15 +30,26 @@ class FakePluginV1Client(object):
self.management_url = kwargs['endpoint']
class TestUndercloudDeploy(TestPluginV1):
class TestDeployUndercloud(TestPluginV1):
def setUp(self):
super(TestUndercloudDeploy, self).setUp()
super(TestDeployUndercloud, self).setUp()
# Get the command object to test
self.cmd = undercloud_deploy.DeployUndercloud(self.app, None)
# Substitute required packages
self.cmd.prerequisites = iter(['foo', 'bar', 'baz'])
undercloud_deploy.DeployUndercloud.heat_pid = mock.MagicMock(
return_value=False)
undercloud_deploy.DeployUndercloud.tht_render = '/twd/templates'
undercloud_deploy.DeployUndercloud.tmp_env_dir = '/twd'
undercloud_deploy.DeployUndercloud.tmp_env_file_name = 'tmp/foo'
undercloud_deploy.DeployUndercloud.heat_launch = mock.MagicMock(
side_effect=(lambda *x, **y: None))
self.tc = self.app.client_manager.tripleoclient = mock.MagicMock()
self.orc = self.tc.local_orchestration = mock.MagicMock()
self.orc.stacks.create = mock.MagicMock(
return_value={'stack': {'id': 'foo'}})
@mock.patch('os.chmod')
@mock.patch('os.path.exists')
@ -99,3 +112,160 @@ class TestUndercloudDeploy(TestPluginV1):
chmod_calls = [mock.call(t_pw_conf_path, 0o600),
mock.call(pw_conf_path, 0o600)]
mock_chmod.assert_has_calls(chmod_calls)
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'get_template_contents', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.environment_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('heatclient.common.template_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
'_setup_heat_environments', autospec=True)
def test_deploy_tripleo_heat_templates_redir(self,
mock_setup_heat_envs,
mock_hc_templ_parse,
mock_hc_env_parse,
mock_hc_get_templ_cont,
mock_hc_process):
parsed_args = self.check_parser(self.cmd,
['--local-ip', '127.0.0.1',
'--templates', '/tmp/thtroot'], [])
mock_setup_heat_envs.return_value = [
'./inside.yaml', '/tmp/thtroot/abs.yaml',
'/tmp/thtroot/puppet/foo.yaml',
'/tmp/thtroot/environments/myenv.yaml',
'/tmp/thtroot42/notouch.yaml',
'../outside.yaml']
self.cmd._deploy_tripleo_heat_templates(self.orc, parsed_args)
mock_hc_process.assert_has_calls([
mock.call(env_path='./inside.yaml'),
mock.call(env_path='/twd/templates/abs.yaml'),
mock.call(env_path='/twd/templates/puppet/foo.yaml'),
mock.call(env_path='/twd/templates/environments/myenv.yaml'),
mock.call(env_path='/tmp/thtroot42/notouch.yaml'),
mock.call(env_path='../outside.yaml')])
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'get_template_contents', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.environment_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('heatclient.common.template_format.'
'parse', autospec=True, return_value=dict())
@mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
'_setup_heat_environments', autospec=True)
@mock.patch('yaml.safe_dump', autospec=True)
@mock.patch('yaml.safe_load', autospec=True)
@mock.patch('six.moves.builtins.open')
@mock.patch('tempfile.NamedTemporaryFile', autospec=True)
def test_deploy_tripleo_heat_templates_rewrite(self,
mock_temp, mock_open,
mock_yaml_load,
mock_yaml_dump,
mock_setup_heat_envs,
mock_hc_templ_parse,
mock_hc_env_parse,
mock_hc_get_templ_cont,
mock_hc_process):
def hc_process(*args, **kwargs):
if 'abs.yaml' in kwargs['env_path']:
raise hc_exc.CommandError
else:
return ({}, {})
mock_hc_process.side_effect = hc_process
parsed_args = self.check_parser(self.cmd,
['--local-ip', '127.0.0.1',
'--templates', '/tmp/thtroot'], [])
rewritten_env = {'resource_registry': {
'OS::Foo::Bar': '/twd/outside.yaml',
'OS::Foo::Baz': '/twd/templates/inside.yaml',
'OS::Foo::Qux': '/twd/templates/abs.yaml',
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
'OS::Foo::Corge': '/twd/templates/puppet/foo.yaml'
}
}
myenv = {'resource_registry': {
'OS::Foo::Bar': '../outside.yaml',
'OS::Foo::Baz': './inside.yaml',
'OS::Foo::Qux': '/tmp/thtroot/abs.yaml',
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
'OS::Foo::Corge': '/tmp/thtroot/puppet/foo.yaml'
}
}
mock_yaml_load.return_value = myenv
mock_setup_heat_envs.return_value = [
'./inside.yaml', '/tmp/thtroot/abs.yaml',
'/tmp/thtroot/puppet/foo.yaml',
'/tmp/thtroot/environments/myenv.yaml',
'../outside.yaml']
self.cmd._deploy_tripleo_heat_templates(self.orc, parsed_args)
mock_yaml_dump.assert_has_calls([mock.call(rewritten_env,
default_flow_style=False)])
@mock.patch('heatclient.common.template_utils.'
'process_environment_and_files', return_value=({}, {}),
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'get_template_contents', return_value=({}, {}),
autospec=True)
@mock.patch('tripleoclient.utils.'
'process_multiple_environments', autospec=True)
@mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
'_update_passwords_env', autospec=True)
@mock.patch('subprocess.check_call', autospec=True)
@mock.patch('netaddr.IPNetwork', autospec=True)
@mock.patch('tempfile.mkdtemp', autospec=True, return_value='/twd')
@mock.patch('shutil.copytree', autospec=True)
def test_setup_heat_environments(self,
mock_copy,
mock_mktemp,
mock_netaddr,
mock_exec,
mock_update_pass_env,
mock_process_multiple_environments,
mock_hc_get_templ_cont,
mock_hc_process):
parsed_args = self.check_parser(self.cmd,
['--local-ip', '127.0.0.1',
'--templates', '/tmp/thtroot',
'--output-dir', '/my',
'-e', '/tmp/thtroot/puppet/foo.yaml',
'-e', '/tmp/thtroot//docker/bar.yaml',
'-e', '/tmp/thtroot42/notouch.yaml',
'-e', '~/custom.yaml',
'-e', 'something.yaml',
'-e', '../../../outside.yaml'], [])
expected_env = [
'/twd/templates/overcloud-resource-registry-puppet.yaml',
mock.ANY,
'/twd/templates/environments/undercloud.yaml',
'/twd/templates/environments/config-download-environment.yaml',
'/twd/templates/environments/deployed-server-noop-ctlplane.yaml',
'/tmp/thtroot/puppet/foo.yaml',
'/tmp/thtroot//docker/bar.yaml',
'/tmp/thtroot42/notouch.yaml',
'~/custom.yaml',
'something.yaml',
'../../../outside.yaml',
mock.ANY]
environment = self.cmd._setup_heat_environments(parsed_args)
self.assertEqual(environment, expected_env)

View File

@ -27,15 +27,18 @@ import six
import socket
import subprocess
import sys
import tempfile
import time
import yaml
from heatclient.common import event_utils
from heatclient.common import template_utils
from heatclient.exc import HTTPNotFound
from osc_lib.i18n import _
from oslo_concurrency import processutils
from six.moves import configparser
from heatclient import exc as hc_exc
from tripleoclient import exceptions
@ -832,6 +835,88 @@ def get_tripleo_ansible_inventory(inventory_file=''):
"Inventory file %s can not be found." % inventory_file)
def process_multiple_environments(created_env_files, tht_root,
user_tht_root, cleanup=True):
log = logging.getLogger(__name__ + ".process_multiple_environments")
env_files = {}
localenv = {}
# Normalize paths for full match checks
user_tht_root = os.path.normpath(user_tht_root)
tht_root = os.path.normpath(tht_root)
for env_path in created_env_files:
log.debug("Processing environment files %s" % env_path)
abs_env_path = os.path.abspath(env_path)
if (abs_env_path.startswith(user_tht_root) and
((user_tht_root + '/') in env_path or
(user_tht_root + '/') in abs_env_path or
user_tht_root == abs_env_path or
user_tht_root == env_path)):
new_env_path = env_path.replace(user_tht_root + '/',
tht_root + '/')
log.debug("Redirecting env file %s to %s"
% (abs_env_path, new_env_path))
env_path = new_env_path
try:
files, env = template_utils.process_environment_and_files(
env_path=env_path)
except hc_exc.CommandError as ex:
# This provides fallback logic so that we can reference files
# inside the resource_registry values that may be rendered via
# j2.yaml templates, where the above will fail because the
# file doesn't exist in user_tht_root, but it is in tht_root
# See bug https://bugs.launchpad.net/tripleo/+bug/1625783
# for details on why this is needed (backwards-compatibility)
log.debug("Error %s processing environment file %s"
% (six.text_type(ex), env_path))
# Use the temporary path as it's possible the environment
# itself was rendered via jinja.
with open(env_path, 'r') as f:
env_map = yaml.safe_load(f)
env_registry = env_map.get('resource_registry', {})
env_dirname = os.path.dirname(os.path.abspath(env_path))
for rsrc, rsrc_path in six.iteritems(env_registry):
# We need to calculate the absolute path relative to
# env_path not cwd (which is what abspath uses).
abs_rsrc_path = os.path.normpath(
os.path.join(env_dirname, rsrc_path))
# If the absolute path matches user_tht_root, rewrite
# a temporary environment pointing at tht_root instead
if (abs_rsrc_path.startswith(user_tht_root) and
((user_tht_root + '/') in abs_rsrc_path or
abs_rsrc_path == user_tht_root)):
new_rsrc_path = abs_rsrc_path.replace(
user_tht_root + '/', tht_root + '/')
log.debug("Rewriting %s %s path to %s"
% (env_path, rsrc, new_rsrc_path))
env_registry[rsrc] = new_rsrc_path
else:
# Skip any resources that are mapping to OS::*
# resource names as these aren't paths
if not rsrc_path.startswith("OS::"):
env_registry[rsrc] = abs_rsrc_path
env_map['resource_registry'] = env_registry
f_name = os.path.basename(os.path.splitext(abs_env_path)[0])
with tempfile.NamedTemporaryFile(dir=tht_root,
prefix="env-%s-" % f_name,
suffix=".yaml",
mode="w",
delete=cleanup) as f:
log.debug("Rewriting %s environment to %s"
% (env_path, f.name))
f.write(yaml.safe_dump(env_map, default_flow_style=False))
f.flush()
files, env = template_utils.process_environment_and_files(
env_path=f.name)
if files:
log.debug("Adding files %s for %s" % (files, env_path))
env_files.update(files)
# 'env' can be a deeply nested dictionary, so a simple update is
# not enough
localenv = template_utils.deep_update(localenv, env)
return env_files, localenv
def run_update_ansible_action(log, clients, nodes, inventory, playbook,
queue, all_playbooks, action):
playbooks = [playbook]

View File

@ -25,7 +25,6 @@ import tempfile
import yaml
from heatclient.common import template_utils
from heatclient import exc as hc_exc
from osc_lib import exceptions as oscexc
from osc_lib.i18n import _
from swiftclient.exceptions import ClientException
@ -181,76 +180,6 @@ class DeployOvercloud(command.Command):
return user_env_path, swift_path
def _process_multiple_environments(self, created_env_files, tht_root,
user_tht_root, cleanup=True):
env_files = {}
localenv = {}
for env_path in created_env_files:
self.log.debug("Processing environment files %s" % env_path)
abs_env_path = os.path.abspath(env_path)
if abs_env_path.startswith(user_tht_root):
new_env_path = abs_env_path.replace(user_tht_root, tht_root)
self.log.debug("Redirecting env file %s to %s"
% (abs_env_path, new_env_path))
env_path = new_env_path
try:
files, env = template_utils.process_environment_and_files(
env_path=env_path)
except hc_exc.CommandError as ex:
# This provides fallback logic so that we can reference files
# inside the resource_registry values that may be rendered via
# j2.yaml templates, where the above will fail because the
# file doesn't exist in user_tht_root, but it is in tht_root
# See bug https://bugs.launchpad.net/tripleo/+bug/1625783
# for details on why this is needed (backwards-compatibility)
self.log.debug("Error %s processing environment file %s"
% (six.text_type(ex), env_path))
# Use the temporary path as it's possible the environment
# itself was rendered via jinja.
with open(env_path, 'r') as f:
env_map = yaml.safe_load(f)
env_registry = env_map.get('resource_registry', {})
env_dirname = os.path.dirname(os.path.abspath(env_path))
for rsrc, rsrc_path in six.iteritems(env_registry):
# We need to calculate the absolute path relative to
# env_path not cwd (which is what abspath uses).
abs_rsrc_path = os.path.normpath(
os.path.join(env_dirname, rsrc_path))
# If the absolute path matches user_tht_root, rewrite
# a temporary environment pointing at tht_root instead
if abs_rsrc_path.startswith(user_tht_root):
new_rsrc_path = abs_rsrc_path.replace(user_tht_root,
tht_root)
self.log.debug("Rewriting %s %s path to %s"
% (env_path, rsrc, new_rsrc_path))
env_registry[rsrc] = new_rsrc_path
else:
# Skip any resources that are mapping to OS::*
# resource names as these aren't paths
if not rsrc_path.startswith("OS::"):
env_registry[rsrc] = abs_rsrc_path
env_map['resource_registry'] = env_registry
f_name = os.path.basename(os.path.splitext(abs_env_path)[0])
with tempfile.NamedTemporaryFile(dir=tht_root,
prefix="env-%s-" % f_name,
suffix=".yaml",
mode="w",
delete=cleanup) as f:
self.log.debug("Rewriting %s environment to %s"
% (env_path, f.name))
f.write(yaml.safe_dump(env_map, default_flow_style=False))
f.flush()
files, env = template_utils.process_environment_and_files(
env_path=f.name)
if files:
self.log.debug("Adding files %s for %s" % (files, env_path))
env_files.update(files)
# 'env' can be a deeply nested dictionary, so a simple update is
# not enough
localenv = template_utils.deep_update(localenv, env)
return env_files, localenv
def _heat_deploy(self, stack, stack_name, template_path, parameters,
env_files, timeout, tht_root, env, update_plan_only,
run_validations, skip_deploy_identifier, plan_env_file):
@ -489,7 +418,7 @@ class DeployOvercloud(command.Command):
created_env_files.extend(parsed_args.environment_files)
self.log.debug("Processing environment files %s" % created_env_files)
env_files, localenv = self._process_multiple_environments(
env_files, localenv = utils.process_multiple_environments(
created_env_files, tht_root, user_tht_root,
cleanup=not parsed_args.no_cleanup)
template_utils.deep_update(env, localenv)

View File

@ -16,17 +16,22 @@
"""Plugin action implementation"""
import copy
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from cryptography import x509
import logging
import netaddr
import os
import yaml
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import serialization
from oslo_config import cfg
from tripleo_common.image import kolla_builder
from tripleoclient import constants
from tripleoclient import utils
from tripleoclient.v1 import undercloud_preflight
import yaml
PARAMETER_MAPPING = {
@ -70,6 +75,15 @@ PATHS = Paths()
# sample config by running "tox -e genconfig" in the project root.
ci_defaults = kolla_builder.container_images_prepare_defaults()
_opts = [
cfg.StrOpt('output_dir',
default=constants.UNDERCLOUD_OUTPUT_DIR,
help=('Directory to output state, processed heat templates, '
'ansible deployment files.'),
),
cfg.BoolOpt('cleanup',
default=False,
help=('Cleanup temporary files'),
),
cfg.StrOpt('deployment_user',
help=('User used to run openstack undercloud install command '
'which will be used to add the user to the docker group, '
@ -652,7 +666,13 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=False):
env_data, registry_overwrites=registry_overwrites)
deploy_args += ['-e', env_file]
deploy_args += ['--output-dir=%s' % os.environ.get('HOME', '')]
if CONF.get('output_dir'):
deploy_args += ['--output-dir=%s' % CONF['output_dir']]
if not os.path.isdir(CONF['output_dir']):
os.mkdir(CONF['output_dir'])
if CONF.get('cleanup'):
deploy_args.append('--cleanup')
if CONF.get('custom_env_files'):
for custom_file in CONF['custom_env_files']:

View File

@ -49,6 +49,7 @@ from six.moves import configparser
from tripleoclient import constants
from tripleoclient import exceptions
from tripleoclient import heat_launcher
from tripleoclient import utils
from tripleo_common.utils import passwords as password_utils
@ -63,6 +64,8 @@ class DeployUndercloud(command.Command):
log = logging.getLogger(__name__ + ".DeployUndercloud")
auth_required = False
heat_pid = None
tht_render = None
tmp_env_dir = None
tmp_env_file_name = None
def _symlink(self, src, dst, tmpd='/tmp'):
@ -225,7 +228,24 @@ class DeployUndercloud(command.Command):
}
return data
def _kill_heat(self):
def _kill_heat(self, parsed_args):
"""Tear down heat installer and temp files
Kill the heat launcher/installer process.
Teardown temp files created in the deployment process,
when cleanup is requested.
"""
if not parsed_args.cleanup and self.tmp_env_dir:
self.log.warning("Not cleaning temporary directory %s"
% self.tmp_env_dir)
elif self.tht_render:
shutil.rmtree(self.tmp_env_dir, ignore_errors=True)
# tht_render is a sub-dir of tmp_env_dir
self.tht_render = None
self.tmp_env_dir = None
if self.tmp_env_file_name:
try:
os.remove(self.tmp_env_file_name)
@ -240,6 +260,9 @@ class DeployUndercloud(command.Command):
def _launch_heat(self, parsed_args):
if not os.path.isdir(parsed_args.output_dir):
os.mkdir(parsed_args.output_dir)
# we do this as root to chown config files properly for docker, etc.
if parsed_args.heat_native:
self.heat_launch = heat_launcher.HeatNativeLauncher(
@ -283,23 +306,38 @@ class DeployUndercloud(command.Command):
return orchestration_client
def _setup_heat_environments(self, parsed_args):
tht_root = parsed_args.templates
# generate jinja templates
"""Process tripleo heat templates with jinja
* Copy --templates content into a temporary working dir
created under the --output_dir path as output_dir/tempwd/templates.
* Process j2 templates there
* Return the environments list for futher processing.
The first two items are reserved for the
overcloud-resource-registry-puppet.yaml and passwords files.
"""
self.tmp_env_dir = tempfile.mkdtemp(prefix='tripleoclient-',
dir=parsed_args.output_dir)
self.tht_render = os.path.join(self.tmp_env_dir, 'templates')
shutil.copytree(parsed_args.templates, self.tht_render, symlinks=True)
# generate jinja templates by its work dir location
self.log.debug("Using roles file %s" % parsed_args.roles_file)
process_templates = os.path.join(tht_root,
process_templates = os.path.join(parsed_args.templates,
'tools/process-templates.py')
args = ['python', process_templates, '--roles-data',
parsed_args.roles_file]
subprocess.check_call(args, cwd=tht_root)
parsed_args.roles_file, '--output-dir', self.tht_render]
subprocess.check_call(args, cwd=self.tht_render)
print("Deploying templates in the directory {0}".format(
os.path.abspath(tht_root)))
os.path.abspath(self.tht_render)))
self.log.debug("Creating Environment file")
environments = []
resource_registry_path = os.path.join(
tht_root, 'overcloud-resource-registry-puppet.yaml')
self.tht_render, 'overcloud-resource-registry-puppet.yaml')
environments.insert(0, resource_registry_path)
# this will allow the user to overwrite passwords with custom envs
@ -307,18 +345,18 @@ class DeployUndercloud(command.Command):
environments.insert(1, pw_file)
undercloud_env_path = os.path.join(
tht_root, 'environments', 'undercloud.yaml')
self.tht_render, 'environments', 'undercloud.yaml')
environments.append(undercloud_env_path)
# use deployed-server because we run os-collect-config locally
deployed_server_env = os.path.join(
tht_root, 'environments',
self.tht_render, 'environments',
'config-download-environment.yaml')
environments.append(deployed_server_env)
# use deployed-server because we run os-collect-config locally
deployed_server_env = os.path.join(
tht_root, 'environments',
self.tht_render, 'environments',
'deployed-server-noop-ctlplane.yaml')
environments.append(deployed_server_env)
@ -364,15 +402,16 @@ class DeployUndercloud(command.Command):
parsed_args):
"""Deploy the fixed templates in TripleO Heat Templates"""
# sets self.tht_render to the temporary work dir after it's done
environments = self._setup_heat_environments(parsed_args)
self.log.debug("Processing environment files")
env_files, env = (
template_utils.process_multiple_environments_and_files(
environments))
self.log.debug("Processing environment files %s" % environments)
env_files, env = utils.process_multiple_environments(
environments, self.tht_render, parsed_args.templates,
cleanup=parsed_args.cleanup)
self.log.debug("Getting template contents")
template_path = os.path.join(parsed_args.templates, 'overcloud.yaml')
template_path = os.path.join(self.tht_render, 'overcloud.yaml')
template_files, template = \
template_utils.get_template_contents(template_path)
@ -454,9 +493,9 @@ class DeployUndercloud(command.Command):
default='undercloud')
parser.add_argument('--output-dir',
dest='output_dir',
help=_("Directory to output state and ansible"
" deployment files."),
default=os.environ.get('HOME', ''))
help=_("Directory to output state, processed heat "
"templates, ansible deployment files."),
default=constants.UNDERCLOUD_OUTPUT_DIR)
parser.add_argument('--output-only',
dest='output_only',
action='store_true',
@ -535,6 +574,11 @@ class DeployUndercloud(command.Command):
default='undercloud',
help=_('Local domain for undercloud and its API endpoints')
)
parser.add_argument(
'--cleanup',
action='store_true', default=False,
help=_('Cleanup temporary files')
)
return parser
def take_action(self, parsed_args):
@ -581,7 +625,7 @@ class DeployUndercloud(command.Command):
parsed_args.stack,
parsed_args.output_dir)
# Kill heat, we're done with it now.
self._kill_heat()
self._kill_heat(parsed_args)
if not parsed_args.output_only:
# Never returns.. We exec() it directly.
self._launch_ansible(ansible_dir)
@ -590,7 +634,7 @@ class DeployUndercloud(command.Command):
print(traceback.format_exception(*sys.exc_info()))
raise
finally:
self._kill_heat()
self._kill_heat(parsed_args)
if not parsed_args.output_only:
# We only get here on error.
print('ERROR: Heat log files: %s' %