Prepare t-h-t for undercloud in a work dir
The issue here is that the undercloud heat installer default uses template in /usr/share and renders the result of the process-templates.py in place. Process templates in a temporary work dir created from the source --templates dir under the --output-dir. Delete generated templates and temp files after the undercloud deployment done or failed, when --cleanup is specified. Ensure the --output-dir exists for all of the cases. The --output-dir and --cleanup may be passed either as the 'undercloud deploy' CLI args, or via the undercloud.conf file, as 'output_dir' and 'cleanup'. The latter way only works for 'undercloud install --use-heat'. Also, share process_multiple_environments in utils lib. Reuse the shared process_multiple_environments for undercloud and overcloud heat installers and add unit tests. Additionally, fix the shared process_multiple_environments redirect/rewrite: only redirect and rewrite paths, if fully matched. Closes-Bug: #1752272 Closes-Bug: #1749683 Change-Id: Ib0c2e3ffd81742441400d27857afae457d71a424 Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
This commit is contained in:
parent
f3948b5532
commit
0b3b55288b
|
@ -0,0 +1,22 @@
|
|||
---
|
||||
fixes:
|
||||
- |
|
||||
Fix undercloud heat installer renders Heat templates in
|
||||
`/usr/share`, which contains t-h-t installed from the package.
|
||||
features:
|
||||
- |
|
||||
New command line arguments `--output-dir` and `--cleanup`
|
||||
define the heat templates processing rules for undercloud:
|
||||
``undercloud deploy --cleanup --output-dir /tmp/tht``.
|
||||
|
||||
The `output_dir` and `cleanup` configuration options
|
||||
for `undercloud.conf` may be used the same way and allow to
|
||||
configure ``undercloud install --use-heat`` behavior.
|
||||
upgrade:
|
||||
- |
|
||||
The default value for `--output-dir` is changed for
|
||||
undercloud heat installer to `$HOME/.undercloud-heat-installer`.
|
||||
The content of the processed heat templates will be persisted
|
||||
under the given path as `$output_dir/$tempdir/templates`, for
|
||||
each run of the undercloud deploy or install commands, unless
|
||||
the `cleanup` mode is requested.
|
|
@ -13,10 +13,14 @@
|
|||
# under the License.
|
||||
#
|
||||
|
||||
import os
|
||||
|
||||
TRIPLEO_HEAT_TEMPLATES = "/usr/share/openstack-tripleo-heat-templates/"
|
||||
OVERCLOUD_YAML_NAME = "overcloud.yaml"
|
||||
OVERCLOUD_ROLES_FILE = "roles_data.yaml"
|
||||
UNDERCLOUD_ROLES_FILE = "roles_data_undercloud.yaml"
|
||||
UNDERCLOUD_OUTPUT_DIR = os.path.join(os.environ.get('HOME'),
|
||||
'.undercloud-heat-installer')
|
||||
OVERCLOUD_NETWORKS_FILE = "network_data.yaml"
|
||||
RHEL_REGISTRATION_EXTRACONFIG_NAME = (
|
||||
"extraconfig/pre_deploy/rhel-registration/")
|
||||
|
@ -27,7 +31,8 @@ USER_ENVIRONMENT = 'user-environment.yaml'
|
|||
USER_PARAMETERS = 'user-environments/tripleoclient-parameters.yaml'
|
||||
|
||||
# This directory may contain additional environments to use during deploy
|
||||
DEFAULT_ENV_DIRECTORY = "~/.tripleo/environments"
|
||||
DEFAULT_ENV_DIRECTORY = os.path.join(os.environ.get('HOME'),
|
||||
'.tripleo', 'environments')
|
||||
|
||||
TRIPLEO_PUPPET_MODULES = "/usr/share/openstack-puppet/modules/"
|
||||
UPGRADE_CONVERGE_FILE = "major-upgrade-converge-docker.yaml"
|
||||
|
|
|
@ -19,6 +19,9 @@ import datetime
|
|||
import mock
|
||||
import os.path
|
||||
import tempfile
|
||||
|
||||
from heatclient import exc as hc_exc
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from unittest import TestCase
|
||||
|
@ -638,3 +641,100 @@ class TestStoreCliParam(TestCase):
|
|||
mock_isdir.return_value = True
|
||||
mock_open.side_effect = IOError()
|
||||
self.assertRaises(IOError, utils.store_cli_param, "command", self.args)
|
||||
|
||||
|
||||
class ProcessMultipleEnvironments(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tht_root = '/twd/templates'
|
||||
self.user_tht_root = '/tmp/thtroot/'
|
||||
self.created_env_files = [
|
||||
'./inside.yaml', '/tmp/thtroot/abs.yaml',
|
||||
'/tmp/thtroot/puppet/foo.yaml',
|
||||
'/tmp/thtroot/environments/myenv.yaml',
|
||||
'/tmp/thtroot42/notouch.yaml',
|
||||
'./tmp/thtroot/notouch2.yaml',
|
||||
'../outside.yaml']
|
||||
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_environment_and_files', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'get_template_contents', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.environment_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
@mock.patch('heatclient.common.template_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
def test_redirect_templates_paths(self,
|
||||
mock_hc_templ_parse,
|
||||
mock_hc_env_parse,
|
||||
mock_hc_get_templ_cont,
|
||||
mock_hc_process):
|
||||
utils.process_multiple_environments(self.created_env_files,
|
||||
self.tht_root,
|
||||
self.user_tht_root)
|
||||
|
||||
mock_hc_process.assert_has_calls([
|
||||
mock.call(env_path='./inside.yaml'),
|
||||
mock.call(env_path='/twd/templates/abs.yaml'),
|
||||
mock.call(env_path='/twd/templates/puppet/foo.yaml'),
|
||||
mock.call(env_path='/twd/templates/environments/myenv.yaml'),
|
||||
mock.call(env_path='/tmp/thtroot42/notouch.yaml'),
|
||||
mock.call(env_path='./tmp/thtroot/notouch2.yaml'),
|
||||
mock.call(env_path='../outside.yaml')])
|
||||
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_environment_and_files', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'get_template_contents', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.environment_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
@mock.patch('heatclient.common.template_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
@mock.patch('yaml.safe_dump', autospec=True)
|
||||
@mock.patch('yaml.safe_load', autospec=True)
|
||||
@mock.patch('six.moves.builtins.open')
|
||||
@mock.patch('tempfile.NamedTemporaryFile', autospec=True)
|
||||
def test_rewrite_env_files(self,
|
||||
mock_temp, mock_open,
|
||||
mock_yaml_load,
|
||||
mock_yaml_dump,
|
||||
mock_hc_templ_parse,
|
||||
mock_hc_env_parse,
|
||||
mock_hc_get_templ_cont,
|
||||
mock_hc_process):
|
||||
|
||||
def hc_process(*args, **kwargs):
|
||||
if 'abs.yaml' in kwargs['env_path']:
|
||||
raise hc_exc.CommandError
|
||||
else:
|
||||
return ({}, {})
|
||||
|
||||
mock_hc_process.side_effect = hc_process
|
||||
rewritten_env = {'resource_registry': {
|
||||
'OS::Foo::Bar': '/twd/outside.yaml',
|
||||
'OS::Foo::Baz': '/twd/templates/inside.yaml',
|
||||
'OS::Foo::Qux': '/twd/templates/abs.yaml',
|
||||
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
|
||||
'OS::Foo::Corge': '/twd/templates/puppet/foo.yaml'
|
||||
}
|
||||
}
|
||||
myenv = {'resource_registry': {
|
||||
'OS::Foo::Bar': '../outside.yaml',
|
||||
'OS::Foo::Baz': './inside.yaml',
|
||||
'OS::Foo::Qux': '/tmp/thtroot/abs.yaml',
|
||||
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
|
||||
'OS::Foo::Corge': '/tmp/thtroot/puppet/foo.yaml'
|
||||
}
|
||||
}
|
||||
mock_yaml_load.return_value = myenv
|
||||
|
||||
utils.process_multiple_environments(self.created_env_files,
|
||||
self.tht_root,
|
||||
self.user_tht_root, False)
|
||||
|
||||
mock_yaml_dump.assert_has_calls([mock.call(rewritten_env,
|
||||
default_flow_style=False)])
|
||||
|
|
|
@ -16,6 +16,8 @@
|
|||
import mock
|
||||
import os
|
||||
|
||||
from heatclient import exc as hc_exc
|
||||
|
||||
from tripleoclient.tests.v1.test_plugin import TestPluginV1
|
||||
|
||||
# Load the plugin init module for the plugin list and show commands
|
||||
|
@ -28,15 +30,26 @@ class FakePluginV1Client(object):
|
|||
self.management_url = kwargs['endpoint']
|
||||
|
||||
|
||||
class TestUndercloudDeploy(TestPluginV1):
|
||||
class TestDeployUndercloud(TestPluginV1):
|
||||
|
||||
def setUp(self):
|
||||
super(TestUndercloudDeploy, self).setUp()
|
||||
super(TestDeployUndercloud, self).setUp()
|
||||
|
||||
# Get the command object to test
|
||||
self.cmd = undercloud_deploy.DeployUndercloud(self.app, None)
|
||||
# Substitute required packages
|
||||
self.cmd.prerequisites = iter(['foo', 'bar', 'baz'])
|
||||
|
||||
undercloud_deploy.DeployUndercloud.heat_pid = mock.MagicMock(
|
||||
return_value=False)
|
||||
undercloud_deploy.DeployUndercloud.tht_render = '/twd/templates'
|
||||
undercloud_deploy.DeployUndercloud.tmp_env_dir = '/twd'
|
||||
undercloud_deploy.DeployUndercloud.tmp_env_file_name = 'tmp/foo'
|
||||
undercloud_deploy.DeployUndercloud.heat_launch = mock.MagicMock(
|
||||
side_effect=(lambda *x, **y: None))
|
||||
|
||||
self.tc = self.app.client_manager.tripleoclient = mock.MagicMock()
|
||||
self.orc = self.tc.local_orchestration = mock.MagicMock()
|
||||
self.orc.stacks.create = mock.MagicMock(
|
||||
return_value={'stack': {'id': 'foo'}})
|
||||
|
||||
@mock.patch('os.chmod')
|
||||
@mock.patch('os.path.exists')
|
||||
|
@ -99,3 +112,160 @@ class TestUndercloudDeploy(TestPluginV1):
|
|||
chmod_calls = [mock.call(t_pw_conf_path, 0o600),
|
||||
mock.call(pw_conf_path, 0o600)]
|
||||
mock_chmod.assert_has_calls(chmod_calls)
|
||||
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_environment_and_files', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'get_template_contents', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.environment_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
@mock.patch('heatclient.common.template_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
@mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
|
||||
'_setup_heat_environments', autospec=True)
|
||||
def test_deploy_tripleo_heat_templates_redir(self,
|
||||
mock_setup_heat_envs,
|
||||
mock_hc_templ_parse,
|
||||
mock_hc_env_parse,
|
||||
mock_hc_get_templ_cont,
|
||||
mock_hc_process):
|
||||
parsed_args = self.check_parser(self.cmd,
|
||||
['--local-ip', '127.0.0.1',
|
||||
'--templates', '/tmp/thtroot'], [])
|
||||
|
||||
mock_setup_heat_envs.return_value = [
|
||||
'./inside.yaml', '/tmp/thtroot/abs.yaml',
|
||||
'/tmp/thtroot/puppet/foo.yaml',
|
||||
'/tmp/thtroot/environments/myenv.yaml',
|
||||
'/tmp/thtroot42/notouch.yaml',
|
||||
'../outside.yaml']
|
||||
|
||||
self.cmd._deploy_tripleo_heat_templates(self.orc, parsed_args)
|
||||
|
||||
mock_hc_process.assert_has_calls([
|
||||
mock.call(env_path='./inside.yaml'),
|
||||
mock.call(env_path='/twd/templates/abs.yaml'),
|
||||
mock.call(env_path='/twd/templates/puppet/foo.yaml'),
|
||||
mock.call(env_path='/twd/templates/environments/myenv.yaml'),
|
||||
mock.call(env_path='/tmp/thtroot42/notouch.yaml'),
|
||||
mock.call(env_path='../outside.yaml')])
|
||||
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_environment_and_files', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'get_template_contents', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.environment_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
@mock.patch('heatclient.common.template_format.'
|
||||
'parse', autospec=True, return_value=dict())
|
||||
@mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
|
||||
'_setup_heat_environments', autospec=True)
|
||||
@mock.patch('yaml.safe_dump', autospec=True)
|
||||
@mock.patch('yaml.safe_load', autospec=True)
|
||||
@mock.patch('six.moves.builtins.open')
|
||||
@mock.patch('tempfile.NamedTemporaryFile', autospec=True)
|
||||
def test_deploy_tripleo_heat_templates_rewrite(self,
|
||||
mock_temp, mock_open,
|
||||
mock_yaml_load,
|
||||
mock_yaml_dump,
|
||||
mock_setup_heat_envs,
|
||||
mock_hc_templ_parse,
|
||||
mock_hc_env_parse,
|
||||
mock_hc_get_templ_cont,
|
||||
mock_hc_process):
|
||||
def hc_process(*args, **kwargs):
|
||||
if 'abs.yaml' in kwargs['env_path']:
|
||||
raise hc_exc.CommandError
|
||||
else:
|
||||
return ({}, {})
|
||||
|
||||
mock_hc_process.side_effect = hc_process
|
||||
|
||||
parsed_args = self.check_parser(self.cmd,
|
||||
['--local-ip', '127.0.0.1',
|
||||
'--templates', '/tmp/thtroot'], [])
|
||||
|
||||
rewritten_env = {'resource_registry': {
|
||||
'OS::Foo::Bar': '/twd/outside.yaml',
|
||||
'OS::Foo::Baz': '/twd/templates/inside.yaml',
|
||||
'OS::Foo::Qux': '/twd/templates/abs.yaml',
|
||||
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
|
||||
'OS::Foo::Corge': '/twd/templates/puppet/foo.yaml'
|
||||
}
|
||||
}
|
||||
myenv = {'resource_registry': {
|
||||
'OS::Foo::Bar': '../outside.yaml',
|
||||
'OS::Foo::Baz': './inside.yaml',
|
||||
'OS::Foo::Qux': '/tmp/thtroot/abs.yaml',
|
||||
'OS::Foo::Quux': '/tmp/thtroot42/notouch.yaml',
|
||||
'OS::Foo::Corge': '/tmp/thtroot/puppet/foo.yaml'
|
||||
}
|
||||
}
|
||||
mock_yaml_load.return_value = myenv
|
||||
|
||||
mock_setup_heat_envs.return_value = [
|
||||
'./inside.yaml', '/tmp/thtroot/abs.yaml',
|
||||
'/tmp/thtroot/puppet/foo.yaml',
|
||||
'/tmp/thtroot/environments/myenv.yaml',
|
||||
'../outside.yaml']
|
||||
|
||||
self.cmd._deploy_tripleo_heat_templates(self.orc, parsed_args)
|
||||
|
||||
mock_yaml_dump.assert_has_calls([mock.call(rewritten_env,
|
||||
default_flow_style=False)])
|
||||
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'process_environment_and_files', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('heatclient.common.template_utils.'
|
||||
'get_template_contents', return_value=({}, {}),
|
||||
autospec=True)
|
||||
@mock.patch('tripleoclient.utils.'
|
||||
'process_multiple_environments', autospec=True)
|
||||
@mock.patch('tripleoclient.v1.undercloud_deploy.DeployUndercloud.'
|
||||
'_update_passwords_env', autospec=True)
|
||||
@mock.patch('subprocess.check_call', autospec=True)
|
||||
@mock.patch('netaddr.IPNetwork', autospec=True)
|
||||
@mock.patch('tempfile.mkdtemp', autospec=True, return_value='/twd')
|
||||
@mock.patch('shutil.copytree', autospec=True)
|
||||
def test_setup_heat_environments(self,
|
||||
mock_copy,
|
||||
mock_mktemp,
|
||||
mock_netaddr,
|
||||
mock_exec,
|
||||
mock_update_pass_env,
|
||||
mock_process_multiple_environments,
|
||||
mock_hc_get_templ_cont,
|
||||
mock_hc_process):
|
||||
|
||||
parsed_args = self.check_parser(self.cmd,
|
||||
['--local-ip', '127.0.0.1',
|
||||
'--templates', '/tmp/thtroot',
|
||||
'--output-dir', '/my',
|
||||
'-e', '/tmp/thtroot/puppet/foo.yaml',
|
||||
'-e', '/tmp/thtroot//docker/bar.yaml',
|
||||
'-e', '/tmp/thtroot42/notouch.yaml',
|
||||
'-e', '~/custom.yaml',
|
||||
'-e', 'something.yaml',
|
||||
'-e', '../../../outside.yaml'], [])
|
||||
expected_env = [
|
||||
'/twd/templates/overcloud-resource-registry-puppet.yaml',
|
||||
mock.ANY,
|
||||
'/twd/templates/environments/undercloud.yaml',
|
||||
'/twd/templates/environments/config-download-environment.yaml',
|
||||
'/twd/templates/environments/deployed-server-noop-ctlplane.yaml',
|
||||
'/tmp/thtroot/puppet/foo.yaml',
|
||||
'/tmp/thtroot//docker/bar.yaml',
|
||||
'/tmp/thtroot42/notouch.yaml',
|
||||
'~/custom.yaml',
|
||||
'something.yaml',
|
||||
'../../../outside.yaml',
|
||||
mock.ANY]
|
||||
|
||||
environment = self.cmd._setup_heat_environments(parsed_args)
|
||||
|
||||
self.assertEqual(environment, expected_env)
|
||||
|
|
|
@ -27,15 +27,18 @@ import six
|
|||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from heatclient.common import event_utils
|
||||
from heatclient.common import template_utils
|
||||
from heatclient.exc import HTTPNotFound
|
||||
from osc_lib.i18n import _
|
||||
from oslo_concurrency import processutils
|
||||
from six.moves import configparser
|
||||
|
||||
from heatclient import exc as hc_exc
|
||||
from tripleoclient import exceptions
|
||||
|
||||
|
||||
|
@ -832,6 +835,88 @@ def get_tripleo_ansible_inventory(inventory_file=''):
|
|||
"Inventory file %s can not be found." % inventory_file)
|
||||
|
||||
|
||||
def process_multiple_environments(created_env_files, tht_root,
|
||||
user_tht_root, cleanup=True):
|
||||
log = logging.getLogger(__name__ + ".process_multiple_environments")
|
||||
env_files = {}
|
||||
localenv = {}
|
||||
# Normalize paths for full match checks
|
||||
user_tht_root = os.path.normpath(user_tht_root)
|
||||
tht_root = os.path.normpath(tht_root)
|
||||
for env_path in created_env_files:
|
||||
log.debug("Processing environment files %s" % env_path)
|
||||
abs_env_path = os.path.abspath(env_path)
|
||||
if (abs_env_path.startswith(user_tht_root) and
|
||||
((user_tht_root + '/') in env_path or
|
||||
(user_tht_root + '/') in abs_env_path or
|
||||
user_tht_root == abs_env_path or
|
||||
user_tht_root == env_path)):
|
||||
new_env_path = env_path.replace(user_tht_root + '/',
|
||||
tht_root + '/')
|
||||
log.debug("Redirecting env file %s to %s"
|
||||
% (abs_env_path, new_env_path))
|
||||
env_path = new_env_path
|
||||
try:
|
||||
files, env = template_utils.process_environment_and_files(
|
||||
env_path=env_path)
|
||||
except hc_exc.CommandError as ex:
|
||||
# This provides fallback logic so that we can reference files
|
||||
# inside the resource_registry values that may be rendered via
|
||||
# j2.yaml templates, where the above will fail because the
|
||||
# file doesn't exist in user_tht_root, but it is in tht_root
|
||||
# See bug https://bugs.launchpad.net/tripleo/+bug/1625783
|
||||
# for details on why this is needed (backwards-compatibility)
|
||||
log.debug("Error %s processing environment file %s"
|
||||
% (six.text_type(ex), env_path))
|
||||
# Use the temporary path as it's possible the environment
|
||||
# itself was rendered via jinja.
|
||||
with open(env_path, 'r') as f:
|
||||
env_map = yaml.safe_load(f)
|
||||
env_registry = env_map.get('resource_registry', {})
|
||||
env_dirname = os.path.dirname(os.path.abspath(env_path))
|
||||
for rsrc, rsrc_path in six.iteritems(env_registry):
|
||||
# We need to calculate the absolute path relative to
|
||||
# env_path not cwd (which is what abspath uses).
|
||||
abs_rsrc_path = os.path.normpath(
|
||||
os.path.join(env_dirname, rsrc_path))
|
||||
# If the absolute path matches user_tht_root, rewrite
|
||||
# a temporary environment pointing at tht_root instead
|
||||
if (abs_rsrc_path.startswith(user_tht_root) and
|
||||
((user_tht_root + '/') in abs_rsrc_path or
|
||||
abs_rsrc_path == user_tht_root)):
|
||||
new_rsrc_path = abs_rsrc_path.replace(
|
||||
user_tht_root + '/', tht_root + '/')
|
||||
log.debug("Rewriting %s %s path to %s"
|
||||
% (env_path, rsrc, new_rsrc_path))
|
||||
env_registry[rsrc] = new_rsrc_path
|
||||
else:
|
||||
# Skip any resources that are mapping to OS::*
|
||||
# resource names as these aren't paths
|
||||
if not rsrc_path.startswith("OS::"):
|
||||
env_registry[rsrc] = abs_rsrc_path
|
||||
env_map['resource_registry'] = env_registry
|
||||
f_name = os.path.basename(os.path.splitext(abs_env_path)[0])
|
||||
with tempfile.NamedTemporaryFile(dir=tht_root,
|
||||
prefix="env-%s-" % f_name,
|
||||
suffix=".yaml",
|
||||
mode="w",
|
||||
delete=cleanup) as f:
|
||||
log.debug("Rewriting %s environment to %s"
|
||||
% (env_path, f.name))
|
||||
f.write(yaml.safe_dump(env_map, default_flow_style=False))
|
||||
f.flush()
|
||||
files, env = template_utils.process_environment_and_files(
|
||||
env_path=f.name)
|
||||
if files:
|
||||
log.debug("Adding files %s for %s" % (files, env_path))
|
||||
env_files.update(files)
|
||||
|
||||
# 'env' can be a deeply nested dictionary, so a simple update is
|
||||
# not enough
|
||||
localenv = template_utils.deep_update(localenv, env)
|
||||
return env_files, localenv
|
||||
|
||||
|
||||
def run_update_ansible_action(log, clients, nodes, inventory, playbook,
|
||||
queue, all_playbooks, action):
|
||||
playbooks = [playbook]
|
||||
|
|
|
@ -25,7 +25,6 @@ import tempfile
|
|||
import yaml
|
||||
|
||||
from heatclient.common import template_utils
|
||||
from heatclient import exc as hc_exc
|
||||
from osc_lib import exceptions as oscexc
|
||||
from osc_lib.i18n import _
|
||||
from swiftclient.exceptions import ClientException
|
||||
|
@ -181,76 +180,6 @@ class DeployOvercloud(command.Command):
|
|||
|
||||
return user_env_path, swift_path
|
||||
|
||||
def _process_multiple_environments(self, created_env_files, tht_root,
|
||||
user_tht_root, cleanup=True):
|
||||
env_files = {}
|
||||
localenv = {}
|
||||
for env_path in created_env_files:
|
||||
self.log.debug("Processing environment files %s" % env_path)
|
||||
abs_env_path = os.path.abspath(env_path)
|
||||
if abs_env_path.startswith(user_tht_root):
|
||||
new_env_path = abs_env_path.replace(user_tht_root, tht_root)
|
||||
self.log.debug("Redirecting env file %s to %s"
|
||||
% (abs_env_path, new_env_path))
|
||||
env_path = new_env_path
|
||||
try:
|
||||
files, env = template_utils.process_environment_and_files(
|
||||
env_path=env_path)
|
||||
except hc_exc.CommandError as ex:
|
||||
# This provides fallback logic so that we can reference files
|
||||
# inside the resource_registry values that may be rendered via
|
||||
# j2.yaml templates, where the above will fail because the
|
||||
# file doesn't exist in user_tht_root, but it is in tht_root
|
||||
# See bug https://bugs.launchpad.net/tripleo/+bug/1625783
|
||||
# for details on why this is needed (backwards-compatibility)
|
||||
self.log.debug("Error %s processing environment file %s"
|
||||
% (six.text_type(ex), env_path))
|
||||
# Use the temporary path as it's possible the environment
|
||||
# itself was rendered via jinja.
|
||||
with open(env_path, 'r') as f:
|
||||
env_map = yaml.safe_load(f)
|
||||
env_registry = env_map.get('resource_registry', {})
|
||||
env_dirname = os.path.dirname(os.path.abspath(env_path))
|
||||
for rsrc, rsrc_path in six.iteritems(env_registry):
|
||||
# We need to calculate the absolute path relative to
|
||||
# env_path not cwd (which is what abspath uses).
|
||||
abs_rsrc_path = os.path.normpath(
|
||||
os.path.join(env_dirname, rsrc_path))
|
||||
# If the absolute path matches user_tht_root, rewrite
|
||||
# a temporary environment pointing at tht_root instead
|
||||
if abs_rsrc_path.startswith(user_tht_root):
|
||||
new_rsrc_path = abs_rsrc_path.replace(user_tht_root,
|
||||
tht_root)
|
||||
self.log.debug("Rewriting %s %s path to %s"
|
||||
% (env_path, rsrc, new_rsrc_path))
|
||||
env_registry[rsrc] = new_rsrc_path
|
||||
else:
|
||||
# Skip any resources that are mapping to OS::*
|
||||
# resource names as these aren't paths
|
||||
if not rsrc_path.startswith("OS::"):
|
||||
env_registry[rsrc] = abs_rsrc_path
|
||||
env_map['resource_registry'] = env_registry
|
||||
f_name = os.path.basename(os.path.splitext(abs_env_path)[0])
|
||||
with tempfile.NamedTemporaryFile(dir=tht_root,
|
||||
prefix="env-%s-" % f_name,
|
||||
suffix=".yaml",
|
||||
mode="w",
|
||||
delete=cleanup) as f:
|
||||
self.log.debug("Rewriting %s environment to %s"
|
||||
% (env_path, f.name))
|
||||
f.write(yaml.safe_dump(env_map, default_flow_style=False))
|
||||
f.flush()
|
||||
files, env = template_utils.process_environment_and_files(
|
||||
env_path=f.name)
|
||||
if files:
|
||||
self.log.debug("Adding files %s for %s" % (files, env_path))
|
||||
env_files.update(files)
|
||||
|
||||
# 'env' can be a deeply nested dictionary, so a simple update is
|
||||
# not enough
|
||||
localenv = template_utils.deep_update(localenv, env)
|
||||
return env_files, localenv
|
||||
|
||||
def _heat_deploy(self, stack, stack_name, template_path, parameters,
|
||||
env_files, timeout, tht_root, env, update_plan_only,
|
||||
run_validations, skip_deploy_identifier, plan_env_file):
|
||||
|
@ -489,7 +418,7 @@ class DeployOvercloud(command.Command):
|
|||
created_env_files.extend(parsed_args.environment_files)
|
||||
|
||||
self.log.debug("Processing environment files %s" % created_env_files)
|
||||
env_files, localenv = self._process_multiple_environments(
|
||||
env_files, localenv = utils.process_multiple_environments(
|
||||
created_env_files, tht_root, user_tht_root,
|
||||
cleanup=not parsed_args.no_cleanup)
|
||||
template_utils.deep_update(env, localenv)
|
||||
|
|
|
@ -16,17 +16,22 @@
|
|||
"""Plugin action implementation"""
|
||||
|
||||
import copy
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography import x509
|
||||
import logging
|
||||
import netaddr
|
||||
import os
|
||||
import yaml
|
||||
|
||||
from cryptography import x509
|
||||
|
||||
from cryptography.hazmat.backends import default_backend
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
|
||||
from oslo_config import cfg
|
||||
from tripleo_common.image import kolla_builder
|
||||
from tripleoclient import constants
|
||||
from tripleoclient import utils
|
||||
|
||||
from tripleoclient.v1 import undercloud_preflight
|
||||
import yaml
|
||||
|
||||
|
||||
PARAMETER_MAPPING = {
|
||||
|
@ -70,6 +75,15 @@ PATHS = Paths()
|
|||
# sample config by running "tox -e genconfig" in the project root.
|
||||
ci_defaults = kolla_builder.container_images_prepare_defaults()
|
||||
_opts = [
|
||||
cfg.StrOpt('output_dir',
|
||||
default=constants.UNDERCLOUD_OUTPUT_DIR,
|
||||
help=('Directory to output state, processed heat templates, '
|
||||
'ansible deployment files.'),
|
||||
),
|
||||
cfg.BoolOpt('cleanup',
|
||||
default=False,
|
||||
help=('Cleanup temporary files'),
|
||||
),
|
||||
cfg.StrOpt('deployment_user',
|
||||
help=('User used to run openstack undercloud install command '
|
||||
'which will be used to add the user to the docker group, '
|
||||
|
@ -652,7 +666,13 @@ def prepare_undercloud_deploy(upgrade=False, no_validations=False):
|
|||
env_data, registry_overwrites=registry_overwrites)
|
||||
deploy_args += ['-e', env_file]
|
||||
|
||||
deploy_args += ['--output-dir=%s' % os.environ.get('HOME', '')]
|
||||
if CONF.get('output_dir'):
|
||||
deploy_args += ['--output-dir=%s' % CONF['output_dir']]
|
||||
if not os.path.isdir(CONF['output_dir']):
|
||||
os.mkdir(CONF['output_dir'])
|
||||
|
||||
if CONF.get('cleanup'):
|
||||
deploy_args.append('--cleanup')
|
||||
|
||||
if CONF.get('custom_env_files'):
|
||||
for custom_file in CONF['custom_env_files']:
|
||||
|
|
|
@ -49,6 +49,7 @@ from six.moves import configparser
|
|||
from tripleoclient import constants
|
||||
from tripleoclient import exceptions
|
||||
from tripleoclient import heat_launcher
|
||||
from tripleoclient import utils
|
||||
|
||||
from tripleo_common.utils import passwords as password_utils
|
||||
|
||||
|
@ -63,6 +64,8 @@ class DeployUndercloud(command.Command):
|
|||
log = logging.getLogger(__name__ + ".DeployUndercloud")
|
||||
auth_required = False
|
||||
heat_pid = None
|
||||
tht_render = None
|
||||
tmp_env_dir = None
|
||||
tmp_env_file_name = None
|
||||
|
||||
def _symlink(self, src, dst, tmpd='/tmp'):
|
||||
|
@ -225,7 +228,24 @@ class DeployUndercloud(command.Command):
|
|||
}
|
||||
return data
|
||||
|
||||
def _kill_heat(self):
|
||||
def _kill_heat(self, parsed_args):
|
||||
"""Tear down heat installer and temp files
|
||||
|
||||
Kill the heat launcher/installer process.
|
||||
Teardown temp files created in the deployment process,
|
||||
when cleanup is requested.
|
||||
|
||||
"""
|
||||
|
||||
if not parsed_args.cleanup and self.tmp_env_dir:
|
||||
self.log.warning("Not cleaning temporary directory %s"
|
||||
% self.tmp_env_dir)
|
||||
elif self.tht_render:
|
||||
shutil.rmtree(self.tmp_env_dir, ignore_errors=True)
|
||||
# tht_render is a sub-dir of tmp_env_dir
|
||||
self.tht_render = None
|
||||
self.tmp_env_dir = None
|
||||
|
||||
if self.tmp_env_file_name:
|
||||
try:
|
||||
os.remove(self.tmp_env_file_name)
|
||||
|
@ -240,6 +260,9 @@ class DeployUndercloud(command.Command):
|
|||
|
||||
def _launch_heat(self, parsed_args):
|
||||
|
||||
if not os.path.isdir(parsed_args.output_dir):
|
||||
os.mkdir(parsed_args.output_dir)
|
||||
|
||||
# we do this as root to chown config files properly for docker, etc.
|
||||
if parsed_args.heat_native:
|
||||
self.heat_launch = heat_launcher.HeatNativeLauncher(
|
||||
|
@ -283,23 +306,38 @@ class DeployUndercloud(command.Command):
|
|||
return orchestration_client
|
||||
|
||||
def _setup_heat_environments(self, parsed_args):
|
||||
tht_root = parsed_args.templates
|
||||
# generate jinja templates
|
||||
"""Process tripleo heat templates with jinja
|
||||
|
||||
* Copy --templates content into a temporary working dir
|
||||
created under the --output_dir path as output_dir/tempwd/templates.
|
||||
* Process j2 templates there
|
||||
* Return the environments list for futher processing.
|
||||
|
||||
The first two items are reserved for the
|
||||
overcloud-resource-registry-puppet.yaml and passwords files.
|
||||
"""
|
||||
|
||||
self.tmp_env_dir = tempfile.mkdtemp(prefix='tripleoclient-',
|
||||
dir=parsed_args.output_dir)
|
||||
self.tht_render = os.path.join(self.tmp_env_dir, 'templates')
|
||||
shutil.copytree(parsed_args.templates, self.tht_render, symlinks=True)
|
||||
|
||||
# generate jinja templates by its work dir location
|
||||
self.log.debug("Using roles file %s" % parsed_args.roles_file)
|
||||
process_templates = os.path.join(tht_root,
|
||||
process_templates = os.path.join(parsed_args.templates,
|
||||
'tools/process-templates.py')
|
||||
args = ['python', process_templates, '--roles-data',
|
||||
parsed_args.roles_file]
|
||||
subprocess.check_call(args, cwd=tht_root)
|
||||
parsed_args.roles_file, '--output-dir', self.tht_render]
|
||||
subprocess.check_call(args, cwd=self.tht_render)
|
||||
|
||||
print("Deploying templates in the directory {0}".format(
|
||||
os.path.abspath(tht_root)))
|
||||
os.path.abspath(self.tht_render)))
|
||||
|
||||
self.log.debug("Creating Environment file")
|
||||
environments = []
|
||||
|
||||
resource_registry_path = os.path.join(
|
||||
tht_root, 'overcloud-resource-registry-puppet.yaml')
|
||||
self.tht_render, 'overcloud-resource-registry-puppet.yaml')
|
||||
environments.insert(0, resource_registry_path)
|
||||
|
||||
# this will allow the user to overwrite passwords with custom envs
|
||||
|
@ -307,18 +345,18 @@ class DeployUndercloud(command.Command):
|
|||
environments.insert(1, pw_file)
|
||||
|
||||
undercloud_env_path = os.path.join(
|
||||
tht_root, 'environments', 'undercloud.yaml')
|
||||
self.tht_render, 'environments', 'undercloud.yaml')
|
||||
environments.append(undercloud_env_path)
|
||||
|
||||
# use deployed-server because we run os-collect-config locally
|
||||
deployed_server_env = os.path.join(
|
||||
tht_root, 'environments',
|
||||
self.tht_render, 'environments',
|
||||
'config-download-environment.yaml')
|
||||
environments.append(deployed_server_env)
|
||||
|
||||
# use deployed-server because we run os-collect-config locally
|
||||
deployed_server_env = os.path.join(
|
||||
tht_root, 'environments',
|
||||
self.tht_render, 'environments',
|
||||
'deployed-server-noop-ctlplane.yaml')
|
||||
environments.append(deployed_server_env)
|
||||
|
||||
|
@ -364,15 +402,16 @@ class DeployUndercloud(command.Command):
|
|||
parsed_args):
|
||||
"""Deploy the fixed templates in TripleO Heat Templates"""
|
||||
|
||||
# sets self.tht_render to the temporary work dir after it's done
|
||||
environments = self._setup_heat_environments(parsed_args)
|
||||
|
||||
self.log.debug("Processing environment files")
|
||||
env_files, env = (
|
||||
template_utils.process_multiple_environments_and_files(
|
||||
environments))
|
||||
self.log.debug("Processing environment files %s" % environments)
|
||||
env_files, env = utils.process_multiple_environments(
|
||||
environments, self.tht_render, parsed_args.templates,
|
||||
cleanup=parsed_args.cleanup)
|
||||
|
||||
self.log.debug("Getting template contents")
|
||||
template_path = os.path.join(parsed_args.templates, 'overcloud.yaml')
|
||||
template_path = os.path.join(self.tht_render, 'overcloud.yaml')
|
||||
template_files, template = \
|
||||
template_utils.get_template_contents(template_path)
|
||||
|
||||
|
@ -454,9 +493,9 @@ class DeployUndercloud(command.Command):
|
|||
default='undercloud')
|
||||
parser.add_argument('--output-dir',
|
||||
dest='output_dir',
|
||||
help=_("Directory to output state and ansible"
|
||||
" deployment files."),
|
||||
default=os.environ.get('HOME', ''))
|
||||
help=_("Directory to output state, processed heat "
|
||||
"templates, ansible deployment files."),
|
||||
default=constants.UNDERCLOUD_OUTPUT_DIR)
|
||||
parser.add_argument('--output-only',
|
||||
dest='output_only',
|
||||
action='store_true',
|
||||
|
@ -535,6 +574,11 @@ class DeployUndercloud(command.Command):
|
|||
default='undercloud',
|
||||
help=_('Local domain for undercloud and its API endpoints')
|
||||
)
|
||||
parser.add_argument(
|
||||
'--cleanup',
|
||||
action='store_true', default=False,
|
||||
help=_('Cleanup temporary files')
|
||||
)
|
||||
return parser
|
||||
|
||||
def take_action(self, parsed_args):
|
||||
|
@ -581,7 +625,7 @@ class DeployUndercloud(command.Command):
|
|||
parsed_args.stack,
|
||||
parsed_args.output_dir)
|
||||
# Kill heat, we're done with it now.
|
||||
self._kill_heat()
|
||||
self._kill_heat(parsed_args)
|
||||
if not parsed_args.output_only:
|
||||
# Never returns.. We exec() it directly.
|
||||
self._launch_ansible(ansible_dir)
|
||||
|
@ -590,7 +634,7 @@ class DeployUndercloud(command.Command):
|
|||
print(traceback.format_exception(*sys.exc_info()))
|
||||
raise
|
||||
finally:
|
||||
self._kill_heat()
|
||||
self._kill_heat(parsed_args)
|
||||
if not parsed_args.output_only:
|
||||
# We only get here on error.
|
||||
print('ERROR: Heat log files: %s' %
|
||||
|
|
Loading…
Reference in New Issue