Download templates from swift before processing with heatclient

Currently we hit an issue in update and upgrade, when heatclieant
try to process the templates and failed because there was some
missing unrendered template files, ie : post.j2.yaml

Co-Authored-By: Steven Hardy <shardy@redhat.com>
Change-Id: If95825e7df5d2c0e6cbe3575d06d57db1e8182da
Closes-Bug: #1624727
changes/47/379547/28
Mathieu Bultel 6 years ago committed by Steven Hardy
parent 4215712bce
commit c683483d0e
  1. 15
      tripleoclient/tests/v1/overcloud_deploy/fakes.py
  2. 215
      tripleoclient/tests/v1/overcloud_deploy/test_overcloud_deploy.py
  3. 124
      tripleoclient/v1/overcloud_deploy.py

@ -107,12 +107,25 @@ class FakeClientWrapper(object):
def __init__(self):
self._instance = mock.Mock()
self.object_store = mock.Mock()
self.object_store = FakeObjectClient()
def messaging_websocket(self, queue_name):
return fakes.FakeWebSocket()
class FakeObjectClient(object):
def __init__(self):
self._instance = mock.Mock()
self.put_object = mock.Mock()
def get_object(self, *args):
return [None, "fake"]
def get_container(self, *args):
return [None, [{"name": "fake"}]]
class TestDeployOvercloud(utils.TestCommand):
def setUp(self):

@ -15,7 +15,6 @@
import fixtures
import os
import shutil
import six
import tempfile
import yaml
@ -54,6 +53,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
self.parameter_defaults_env_file = (
tempfile.NamedTemporaryFile(mode='w', delete=False).name)
self.tmp_dir = self.useFixture(fixtures.TempDir())
def tearDown(self):
super(TestDeployOvercloud, self).tearDown()
@ -79,8 +79,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.remove_known_hosts', autospec=True)
@mock.patch('tripleoclient.utils.wait_for_stack_ready',
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files', autospec=True)
@mock.patch('heatclient.common.template_utils.get_template_contents',
autospec=True)
@mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@ -94,10 +92,11 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.create_keystone_credential',
autospec=True)
@mock.patch('time.time', autospec=True)
def test_tht_scale(self, mock_time, mock_creds, mock_uuid1,
@mock.patch('shutil.copytree', autospec=True)
def test_tht_scale(self, mock_copy, mock_time, mock_creds, mock_uuid1,
mock_check_hypervisor_stats, mock_get_key,
mock_create_env, generate_certs_mock,
mock_get_template_contents, mock_process_multiple_env,
mock_get_template_contents,
wait_for_stack_ready_mock,
mock_remove_known_hosts, mock_keystone_initialize,
mock_sleep, mock_setup_endpoints,
@ -143,8 +142,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
"id": "network id"
}
mock_create_env.return_value = "/fake/path"
mock_env = fakes.create_env()
mock_process_multiple_env.return_value = [{}, mock_env]
mock_get_template_contents.return_value = [{}, "template"]
wait_for_stack_ready_mock.return_value = True
@ -221,7 +218,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
template_object=constants.OVERCLOUD_YAML_NAME)
mock_create_tempest_deployer_input.assert_called_with()
mock_process_multiple_env.assert_called_with([])
@mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
@ -250,7 +246,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.wait_for_stack_ready',
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files', autospec=True)
'process_environment_and_files', autospec=True)
@mock.patch('heatclient.common.template_utils.get_template_contents',
autospec=True)
@mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@ -264,10 +260,13 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.create_keystone_credential',
autospec=True)
@mock.patch('time.time', autospec=True)
def test_tht_deploy(self, mock_time, mock_creds, mock_uuid1,
@mock.patch('shutil.copytree', autospec=True)
@mock.patch('tempfile.mkdtemp', autospec=True)
def test_tht_deploy(self, mock_tmpdir, mock_copy, mock_time, mock_creds,
mock_uuid1,
mock_check_hypervisor_stats, mock_get_key,
mock_create_env, generate_certs_mock,
mock_get_template_contents, mock_process_multiple_env,
mock_get_template_contents, mock_process_env,
wait_for_stack_ready_mock,
mock_remove_known_hosts, mock_keystone_initialize,
mock_sleep, mock_setup_endpoints,
@ -285,6 +284,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
('ceph_storage_scale', 3)
]
mock_tmpdir.return_value = "/tmp/tht"
mock_uuid1.return_value = "uuid"
mock_creds.return_value = "key"
mock_time.return_value = 123456789
@ -317,7 +317,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
}
mock_create_env.return_value = "/fake/path"
mock_env = fakes.create_env()
mock_process_multiple_env.return_value = [{}, mock_env]
mock_process_env.return_value = [{}, mock_env]
mock_get_template_contents.return_value = [{}, "template"]
wait_for_stack_ready_mock.return_value = True
@ -396,12 +396,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
template_object=constants.OVERCLOUD_YAML_NAME)
mock_create_tempest_deployer_input.assert_called_with()
mock_process_multiple_env.assert_called_with(['/fake/path'])
mock_process_env.assert_called_with(env_path='/fake/path')
mock_validate_args.assert_called_once_with(parsed_args)
mock_tarball.create_tarball.assert_called_with(
'/usr/share/openstack-tripleo-heat-templates', mock.ANY)
'/tmp/tht/tripleo-heat-templates', mock.ANY)
mock_tarball.tarball_extract_to_swift_container.assert_called_with(
clients.tripleoclient.object_store, mock.ANY, 'overcloud')
@ -423,8 +423,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.remove_known_hosts', autospec=True)
@mock.patch('tripleoclient.utils.wait_for_stack_ready',
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files', autospec=True)
@mock.patch('heatclient.common.template_utils.get_template_contents',
autospec=True)
@mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@ -434,11 +432,12 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.get_config_value', autospec=True)
@mock.patch('tripleoclient.utils.check_hypervisor_stats',
autospec=True)
def test_deploy_custom_templates(self, mock_check_hypervisor_stats,
@mock.patch('shutil.copytree', autospec=True)
def test_deploy_custom_templates(self, mock_copy,
mock_check_hypervisor_stats,
mock_get_key,
mock_create_env, generate_certs_mock,
mock_get_template_contents,
mock_process_multiple_env,
wait_for_stack_ready_mock,
mock_remove_known_hosts,
mock_keystone_initialize,
@ -472,8 +471,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
"id": "network id"
}
mock_create_env.return_value = "/fake/path"
mock_env = fakes.create_env()
mock_process_multiple_env.return_value = [{}, mock_env]
mock_get_template_contents.return_value = [{}, "template"]
wait_for_stack_ready_mock.return_value = True
@ -501,7 +498,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
template_object=constants.OVERCLOUD_YAML_NAME)
mock_create_tempest_deployer_input.assert_called_with()
mock_process_multiple_env.assert_called_with([])
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'set_overcloud_passwords', autospec=True)
@ -539,7 +535,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_update_parameters', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_heat_deploy', autospec=True)
def test_environment_dirs(self, mock_deploy_heat,
@mock.patch('shutil.copytree', autospec=True)
def test_environment_dirs(self, mock_copy, mock_deploy_heat,
mock_update_parameters, mock_post_config,
mock_utils_check_nodes, mock_utils_endpoint,
mock_utils_createrc, mock_utils_tempest,
@ -553,25 +550,29 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_update_parameters.return_value = {}
mock_utils_endpoint.return_value = 'foo.bar'
tmp_dir = self.useFixture(fixtures.TempDir())
test_env = os.path.join(tmp_dir.path, 'foo1.yaml')
test_env = os.path.join(self.tmp_dir.path, 'foo1.yaml')
env_dirs = [os.path.join(os.environ.get('HOME', ''), '.tripleo',
'environments'), tmp_dir.path]
'environments'), self.tmp_dir.path]
with open(test_env, 'w') as temp_file:
temp_file.write('#just a comment')
temp_file.write('resource_registry:\n Test: OS::Heat::None')
arglist = ['--templates', '--environment-directory', tmp_dir.path]
arglist = ['--templates', '--environment-directory', self.tmp_dir.path]
verifylist = [
('templates', '/usr/share/openstack-tripleo-heat-templates/'),
('environment_directories', env_dirs),
]
def assertEqual(*args):
self.assertEqual(*args)
def _fake_heat_deploy(self, stack, stack_name, template_path,
parameters, environments, timeout, tht_root,
env):
assert test_env in environments
assertEqual(
{'parameter_defaults': {},
'resource_registry': {'Test': u'OS::Heat::None'}}, env)
mock_deploy_heat.side_effect = _fake_heat_deploy
@ -591,7 +592,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_update_parameters', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_heat_deploy', autospec=True)
def test_environment_dirs_env(self, mock_deploy_heat,
@mock.patch('shutil.copytree', autospec=True)
def test_environment_dirs_env(self, mock_copy, mock_deploy_heat,
mock_update_parameters, mock_post_config,
mock_utils_check_nodes, mock_utils_endpoint,
mock_utils_createrc, mock_utils_tempest,
@ -605,23 +607,26 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_update_parameters.return_value = {}
mock_utils_endpoint.return_value = 'foo.bar'
tmp_dir = tempfile.mkdtemp()
test_env = os.path.join(tmp_dir, 'foo2.yaml')
self.addCleanup(shutil.rmtree, tmp_dir)
test_env = self.tmp_dir.join('foo2.yaml')
with open(test_env, 'w') as temp_file:
temp_file.write('#just a comment')
temp_file.write('resource_registry:\n Test: OS::Heat::None')
arglist = ['--templates']
verifylist = [
('templates', '/usr/share/openstack-tripleo-heat-templates/'),
]
os.environ['TRIPLEO_ENVIRONMENT_DIRECTORY'] = tmp_dir
os.environ['TRIPLEO_ENVIRONMENT_DIRECTORY'] = self.tmp_dir.path
def assertEqual(*args):
self.assertEqual(*args)
def _fake_heat_deploy(self, stack, stack_name, template_path,
parameters, environments, timeout, tht_root,
env):
assert test_env in environments
assertEqual(
{'parameter_defaults': {},
'resource_registry': {'Test': u'OS::Heat::None'}}, env)
mock_deploy_heat.side_effect = _fake_heat_deploy
@ -636,7 +641,8 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.get_overcloud_endpoint', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'_deploy_tripleo_heat_templates', autospec=True)
def test_rhel_reg_params_provided(self, mock_deploy_tht,
@mock.patch('shutil.copytree', autospec=True)
def test_rhel_reg_params_provided(self, mock_copytree, mock_deploy_tht,
mock_oc_endpoint,
mock_create_ocrc,
mock_set_oc_passwords,
@ -682,7 +688,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.wait_for_stack_ready',
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files', autospec=True)
'process_environment_and_files', autospec=True)
@mock.patch('heatclient.common.template_utils.get_template_contents',
autospec=True)
@mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@ -692,11 +698,15 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.get_config_value', autospec=True)
@mock.patch('tripleoclient.utils.check_hypervisor_stats',
autospec=True)
def test_deploy_rhel_reg(self, mock_check_hypervisor_stats,
@mock.patch('shutil.copytree', autospec=True)
@mock.patch('tempfile.mkdtemp', autospec=True)
@mock.patch('shutil.rmtree', autospec=True)
def test_deploy_rhel_reg(self, mock_rmtree, mock_tmpdir, mock_copy,
mock_check_hypervisor_stats,
mock_get_key,
mock_create_env, generate_certs_mock,
mock_get_template_contents,
mock_process_multiple_env,
mock_process_env,
wait_for_stack_ready_mock,
mock_remove_known_hosts,
mock_keystone_initialize,
@ -721,8 +731,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
('reg_activation_key', 'super-awesome-key')
]
mock_tmpdir.return_value = None
mock_tmpdir.return_value = '/tmp/tht'
mock_generate_overcloud_passwords.return_value = self._get_passwords()
mock_process_multiple_env.return_value = [{}, fakes.create_env()]
mock_process_env.return_value = [{}, fakes.create_env()]
mock_get_template_contents.return_value = [{}, "template"]
wait_for_stack_ready_mock.return_value = True
@ -753,15 +765,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
self.cmd.take_action(parsed_args)
args, kwargs = mock_process_multiple_env.call_args
self.assertIn(
'/usr/share/openstack-tripleo-heat-templates/extraconfig/pre_dep'
'loy/rhel-registration/rhel-registration-resource-registry.yaml',
args[0])
self.assertIn(
'/usr/share/openstack-tripleo-heat-templates/extraconfig/pre_dep'
'loy/rhel-registration/environment-rhel-registration.yaml',
args[0])
tht_prefix = ('/tmp/tht/tripleo-heat-templates/extraconfig/'
'pre_deploy/rhel-registration/')
calls = [
mock.call(env_path=tht_prefix +
'rhel-registration-resource-registry.yaml'),
mock.call(env_path=tht_prefix +
'environment-rhel-registration.yaml')]
mock_process_env.assert_has_calls(calls)
def test_validate_args_correct(self):
arglist = ['--templates',
@ -792,7 +803,9 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
self.cmd._validate_args,
parsed_args)
def test_validate_args_missing_environment_files(self):
@mock.patch('tripleoclient.tests.v1.overcloud_deploy.fakes.'
'FakeObjectClient.get_object', autospec=True)
def test_validate_args_missing_environment_files(self, mock_obj):
arglist = ['--templates',
'-e', 'nonexistent.yaml']
verifylist = [
@ -800,12 +813,33 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
('environment_files', ['nonexistent.yaml']),
]
mock_obj.side_effect = ObjectClientException(mock.Mock(
'/fake/path not found'))
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.assertRaises(oscexc.CommandError,
self.cmd._validate_args,
parsed_args)
@mock.patch('os.path.isfile', autospec=True)
def test_validate_args_missing_rendered_files(self, mock_isfile):
tht_path = '/usr/share/openstack-tripleo-heat-templates/'
env_path = os.path.join(tht_path, 'noexist.yaml')
arglist = ['--templates',
'-e', env_path]
verifylist = [
('templates', '/usr/share/openstack-tripleo-heat-templates/'),
('environment_files', [env_path]),
]
mock_isfile.side_effect = [False, True]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd._validate_args(parsed_args)
calls = [mock.call(env_path),
mock.call(env_path.replace(".yaml", ".j2.yaml"))]
mock_isfile.assert_has_calls(calls)
def test_validate_args_no_tunnel_type(self):
arglist = ['--templates',
'--neutron-network-type', 'nettype']
@ -953,7 +987,10 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
'_heat_deploy', autospec=True)
@mock.patch('tripleoclient.v1.overcloud_deploy.DeployOvercloud.'
'set_overcloud_passwords', autospec=True)
def test_answers_file(self,
@mock.patch('shutil.copytree', autospec=True)
@mock.patch('tempfile.mkdtemp', autospec=True)
@mock.patch('shutil.rmtree', autospec=True)
def test_answers_file(self, mock_rmtree, mock_tmpdir, mock_copy,
mock_set_overcloud_passwords,
mock_heat_deploy,
mock_oc_endpoint,
@ -966,32 +1003,41 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
workflow_client.action_executions.create.return_value = mock.MagicMock(
output='{"result":[]}')
mock_tmpdir.return_value = self.tmp_dir.path
mock_rmtree.return_value = None
network_client = clients.network
network_client.stacks.get.return_value = None
net = network_client.api.find_attr('networks', 'ctlplane')
net.configure_mock(__getitem__=lambda x, y: 'testnet')
with tempfile.NamedTemporaryFile(mode="w+t") as answerfile:
with open('/tmp/environment.yaml', "w+t") as environmentfile:
test_env = self.tmp_dir.join('foo1.yaml')
with open(test_env, 'w') as temp_file:
temp_file.write('resource_registry:\n Test: OS::Heat::None')
test_env2 = self.tmp_dir.join('foo2.yaml')
with open(test_env2, 'w') as temp_file:
temp_file.write('resource_registry:\n Test2: OS::Heat::None')
test_answerfile = self.tmp_dir.join('answerfile')
with open(test_answerfile, 'w') as answerfile:
yaml.dump(
{'templates': '/dev/null',
'environments': ['/tmp/foo3.yaml']
{'templates':
'/usr/share/openstack-tripleo-heat-templates/',
'environments': [test_env]
},
answerfile
)
answerfile.flush()
arglist = ['--answers-file', answerfile.name,
'--environment-file', environmentfile.name,
'--block-storage-scale', '3']
verifylist = [
('answers_file', answerfile.name),
('environment_files', [environmentfile.name]),
('block_storage_scale', 3)
]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
arglist = ['--answers-file', test_answerfile,
'--environment-file', test_env2,
'--block-storage-scale', '3']
verifylist = [
('answers_file', test_answerfile),
('environment_files', [test_env2]),
('block_storage_scale', 3)]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
self.cmd.take_action(parsed_args)
self.assertTrue(mock_heat_deploy.called)
self.assertTrue(mock_oc_endpoint.called)
@ -1000,12 +1046,15 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
# Check that Heat was called with correct parameters:
call_args = mock_heat_deploy.call_args[0]
self.assertEqual(call_args[3], '/dev/null/overcloud.yaml')
self.assertIn('/tmp/foo3.yaml', call_args[5])
self.assertIn('/tmp/environment.yaml', call_args[5])
foo_index = call_args[5].index('/tmp/foo3.yaml')
env_index = call_args[5].index('/tmp/environment.yaml')
self.assertGreater(env_index, foo_index)
self.assertEqual(call_args[3],
self.tmp_dir.join(
'tripleo-heat-templates/overcloud.yaml'))
self.assertEqual(call_args[7],
self.tmp_dir.join('tripleo-heat-templates'))
self.assertIn('Test', call_args[8]['resource_registry'])
self.assertIn('Test2', call_args[8]['resource_registry'])
self.assertEqual(
3, call_args[8]['parameter_defaults']['BlockStorageCount'])
mock_create_tempest_deployer_input.assert_called_with()
@ -1101,16 +1150,18 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.generate_overcloud_passwords')
@mock.patch('tripleoclient.utils.create_overcloudrc')
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files', autospec=True)
'process_environment_and_files', autospec=True)
@mock.patch('heatclient.common.template_utils.get_template_contents',
autospec=True)
@mock.patch('tripleoclient.utils.create_environment_file',
autospec=True)
@mock.patch('tripleoclient.utils.get_config_value', autospec=True)
def test_ntp_server_mandatory(self, mock_get_key,
@mock.patch('shutil.copytree', autospec=True)
def test_ntp_server_mandatory(self, mock_copy,
mock_get_key,
mock_create_env,
mock_get_template_contents,
mock_process_multiple_env,
mock_process_env,
mock_create_overcloudrc,
mock_generate_overcloud_passwords,
mock_create_parameters_env,
@ -1141,7 +1192,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
mock_create_env.return_value = "/fake/path"
mock_env = fakes.create_env()
mock_process_multiple_env.return_value = [{}, mock_env]
mock_process_env.return_value = [{}, mock_env]
mock_get_template_contents.return_value = [{}, "template"]
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
@ -1174,7 +1225,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.wait_for_stack_ready',
autospec=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files', autospec=True)
'process_environment_and_files', autospec=True)
@mock.patch('heatclient.common.template_utils.get_template_contents',
autospec=True)
@mock.patch('os_cloud_config.keystone_pki.generate_certs_into_json',
@ -1188,12 +1239,14 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
@mock.patch('tripleoclient.utils.create_keystone_credential',
autospec=True)
@mock.patch('time.time', autospec=True)
def test_tht_deploy_with_ntp(self, mock_time, mock_creds, mock_uuid1,
@mock.patch('shutil.copytree', autospec=True)
def test_tht_deploy_with_ntp(self, mock_copy, mock_time, mock_creds,
mock_uuid1,
mock_check_hypervisor_stats,
mock_get_key, mock_create_env,
generate_certs_mock,
mock_get_template_contents,
mock_process_multiple_env,
mock_process_env,
wait_for_stack_ready_mock,
mock_remove_known_hosts,
mock_keystone_initialize,
@ -1253,7 +1306,7 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
}
mock_create_env.return_value = "/fake/path"
mock_env = fakes.create_env_with_ntp()
mock_process_multiple_env.return_value = [{}, mock_env]
mock_process_env.return_value = [{}, mock_env]
mock_get_template_contents.return_value = [{}, "template"]
wait_for_stack_ready_mock.return_value = True
@ -1328,6 +1381,6 @@ class TestDeployOvercloud(fakes.TestDeployOvercloud):
template_object=constants.OVERCLOUD_YAML_NAME)
mock_create_tempest_deployer_input.assert_called_with()
mock_process_multiple_env.assert_called_with(['/fake/path'])
mock_process_env.assert_called_with(env_path='/fake/path')
mock_validate_args.assert_called_once_with(parsed_args)

@ -21,12 +21,15 @@ import logging
import os
import os.path
import re
import shutil
import six
import tempfile
import time
import uuid
import yaml
from heatclient.common import template_utils
from heatclient import exc as hc_exc
from keystoneclient import exceptions as kscexc
from os_cloud_config import keystone
from os_cloud_config import keystone_pki
@ -219,22 +222,46 @@ class DeployOvercloud(command.Command):
parameter_defaults = {"parameter_defaults": parameters}
return parameter_defaults
def _process_multiple_environments(self, created_env_files, added_files,
tht_root, user_tht_root):
env_files = {}
localenv = {}
for env_path in created_env_files:
self.log.debug("Processing environment files %s" % env_path)
abs_env_path = os.path.abspath(env_path)
if abs_env_path.startswith(user_tht_root):
new_env_path = abs_env_path.replace(user_tht_root, tht_root)
self.log.debug("Redirecting env file %s to %s"
% (abs_env_path, new_env_path))
env_path = new_env_path
try:
files, env = template_utils.process_environment_and_files(
env_path=env_path)
except hc_exc.CommandError as ex:
self.log.debug("Error %s processing environment file %s"
% (six.text_type(ex), env_path))
# FIXME(shardy) We need logic here to handle the case described
# in https://bugs.launchpad.net/tripleo/+bug/1625783 so that
# resource_registry contents are redirected to tmpdir tht_root
raise
if files:
self.log.debug("Adding files %s for %s" % (files, env_path))
env_files.update(files)
# 'env' can be a deeply nested dictionary, so a simple update is
# not enough
localenv = template_utils.deep_update(localenv, env)
return env_files, localenv
def _heat_deploy(self, stack, stack_name, template_path, parameters,
created_env_files, timeout, tht_root, env):
env_files, timeout, tht_root, env):
"""Verify the Baremetal nodes are available and do a stack update"""
clients = self.app.client_manager
workflow_client = clients.workflow_engine
self.log.debug("Processing environment files %s" % created_env_files)
env_files, localenv = (
template_utils.process_multiple_environments_and_files(
created_env_files))
# Command line has more precedence than env files
template_utils.deep_update(localenv, env)
if stack:
update.add_breakpoints_cleanup_into_env(localenv)
update.add_breakpoints_cleanup_into_env(env)
self.log.debug("Getting template contents from plan %s" % stack_name)
# We need to reference the plan here, not the local
@ -259,7 +286,7 @@ class DeployOvercloud(command.Command):
number_controllers = int(parameters.get('ControllerCount', 0))
if number_controllers > 1:
if not localenv.get('parameter_defaults').get('NtpServer'):
if not env.get('parameter_defaults').get('NtpServer'):
raise exceptions.InvalidConfiguration(
'Specify --ntp-server as parameter or NtpServer in '
'environments when using multiple controllers '
@ -270,7 +297,7 @@ class DeployOvercloud(command.Command):
moved_files = self._upload_missing_files(
stack_name, objectclient, files, tht_root)
self._process_and_upload_environment(
stack_name, objectclient, localenv, moved_files, tht_root,
stack_name, objectclient, env, moved_files, tht_root,
workflow_client)
deployment.deploy_and_wait(self.log, clients, stack, stack_name,
@ -381,7 +408,47 @@ class DeployOvercloud(command.Command):
return file_relocation
def _deploy_tripleo_heat_templates(self, stack, parsed_args):
def _download_missing_files_from_plan(self, tht_dir, plan_name):
# get and download missing files into tmp directory
clients = self.app.client_manager
objectclient = clients.tripleoclient.object_store
plan_list = objectclient.get_container(plan_name)
plan_filenames = [f['name'] for f in plan_list[1]]
added_files = {}
for pf in plan_filenames:
file_path = os.path.join(tht_dir, pf)
if not os.path.isfile(file_path):
self.log.debug("Missing in templates directory, downloading \
%s from swift into %s" % (pf, file_path))
if not os.path.exists(os.path.dirname(file_path)):
os.makedirs(os.path.dirname(file_path))
with open(file_path, 'w') as f:
f.write(objectclient.get_object(plan_name, pf)[1])
added_files[pf] = file_path
self.log.debug("added_files = %s" % added_files)
return added_files
def _deploy_tripleo_heat_templates_tmpdir(self, stack, parsed_args):
# copy tht_root to temporary directory because we need to
# download any missing (e.g j2 rendered) files from the plan
tht_root = os.path.abspath(parsed_args.templates)
tht_tmp = tempfile.mkdtemp(prefix='tripleoclient-')
new_tht_root = "%s/tripleo-heat-templates" % tht_tmp
self.log.debug("Creating temporary templates tree in %s"
% new_tht_root)
try:
shutil.copytree(tht_root, new_tht_root, symlinks=True)
self._deploy_tripleo_heat_templates(stack, parsed_args,
new_tht_root, tht_root)
finally:
if parsed_args.no_cleanup:
self.log.warning("Not cleaning temporary directory %s"
% tht_tmp)
else:
shutil.rmtree(tht_tmp)
def _deploy_tripleo_heat_templates(self, stack, parsed_args,
tht_root, user_tht_root):
"""Deploy the fixed templates in TripleO Heat Templates"""
clients = self.app.client_manager
network_client = clients.network
@ -390,8 +457,6 @@ class DeployOvercloud(command.Command):
parameters = self._update_parameters(
parsed_args, network_client, stack)
tht_root = os.path.abspath(parsed_args.templates)
plans = plan_management.list_deployment_plans(workflow_client)
# TODO(d0ugal): We need to put a more robust strategy in place here to
@ -405,6 +470,10 @@ class DeployOvercloud(command.Command):
plan_management.create_plan_from_templates(
clients, parsed_args.stack, tht_root, parsed_args.roles_file)
# Get any missing (e.g j2 rendered) files from the plan to tht_root
added_files = self._download_missing_files_from_plan(
tht_root, parsed_args.stack)
print("Deploying templates in the directory {0}".format(
os.path.abspath(tht_root)))
@ -433,18 +502,25 @@ class DeployOvercloud(command.Command):
if parsed_args.environment_files:
created_env_files.extend(parsed_args.environment_files)
self.log.debug("Processing environment files %s" % created_env_files)
env_files, localenv = self._process_multiple_environments(
created_env_files, added_files, tht_root, user_tht_root)
# Command line has more precedence than env files
template_utils.deep_update(localenv, env)
self._try_overcloud_deploy_with_compat_yaml(
tht_root, stack, parsed_args.stack, parameters, created_env_files,
parsed_args.timeout, env)
tht_root, stack, parsed_args.stack, parameters, env_files,
parsed_args.timeout, localenv)
def _try_overcloud_deploy_with_compat_yaml(self, tht_root, stack,
stack_name, parameters,
created_env_files, timeout,
env_files, timeout,
env):
overcloud_yaml = os.path.join(tht_root, constants.OVERCLOUD_YAML_NAME)
try:
self._heat_deploy(stack, stack_name, overcloud_yaml,
parameters, created_env_files, timeout,
parameters, env_files, timeout,
tht_root, env)
except ClientException as e:
messages = 'Failed to deploy: %s' % str(e)
@ -631,7 +707,11 @@ class DeployOvercloud(command.Command):
nonexisting_envs = []
for env_file in parsed_args.environment_files:
if not os.path.isfile(env_file):
nonexisting_envs.append(env_file)
# Tolerate missing file if there's a j2.yaml file that will
# be rendered in the plan but not available locally (yet)
if not os.path.isfile(env_file.replace(".yaml",
".j2.yaml")):
nonexisting_envs.append(env_file)
if nonexisting_envs:
raise oscexc.CommandError(
"Error: The following files were not found: {0}".format(
@ -948,6 +1028,10 @@ class DeployOvercloud(command.Command):
help=_('Roles file, overrides the default %s in the --templates '
'directory') % constants.OVERCLOUD_ROLES_FILE
)
parser.add_argument(
'--no-cleanup', action='store_true',
help=_('Don\'t cleanup temporary files, just log their location')
)
# TODO(bnemec): In Ocata or later, remove this group and just leave
# --validation-errors-nonfatal
error_group = parser.add_mutually_exclusive_group()
@ -1101,7 +1185,7 @@ class DeployOvercloud(command.Command):
print("Validation Finished")
return
self._deploy_tripleo_heat_templates(stack, parsed_args)
self._deploy_tripleo_heat_templates_tmpdir(stack, parsed_args)
# Get a new copy of the stack after stack update/create. If it was
# a create then the previous stack object would be None.

Loading…
Cancel
Save