Remove Heat API dependency for overcloud export

Update the overcloud export command to work without a dependency on Heat
API. The export data is updated to create the export data based solely
on saved stack output data and the config-download generated
overcloud.json so that Heat is not needed to generate the export at all.

Change-Id: I0bd4b4a0e33184e0f4d731899138bf4063f3cdac
Signed-off-by: James Slagle <jslagle@redhat.com>
This commit is contained in:
James Slagle 2021-07-16 08:16:04 -04:00
parent 035827f3db
commit d7af2f5482
5 changed files with 126 additions and 168 deletions

View File

@ -29,12 +29,12 @@ from tripleoclient import utils as oooutils
LOG = logging.getLogger(__name__ + ".utils")
def export_passwords(heat, stack, excludes=True):
def export_passwords(working_dir, stack, excludes=True):
"""For each password, check if it's excluded, then check if there's a user
defined value from parameter_defaults, and if not use the value from the
generated passwords.
:param heat: tht client
:type heat: Client
:param working_dir: Working dir for the deployment
:type working_dir: string
:param stack: stack name for password generator
:type stack: string
:param excludes: filter the passwords or not, defaults to `True`
@ -48,8 +48,13 @@ def export_passwords(heat, stack, excludes=True):
if re.match(pattern, password, re.I):
return True
passwords_file = os.path.join(
working_dir,
constants.PASSWORDS_ENV_FORMAT.format(stack))
with open(passwords_file) as f:
passwords_env = yaml.safe_load(f.read())
generated_passwords = plan_utils.generate_passwords(
heat=heat, container=stack)
passwords_env=passwords_env)
filtered_passwords = generated_passwords.copy()
@ -61,14 +66,14 @@ def export_passwords(heat, stack, excludes=True):
return filtered_passwords
def export_stack(heat, stack, should_filter=False,
def export_stack(working_dir, stack, should_filter=False,
config_download_dir=constants.DEFAULT_WORK_DIR):
"""Export stack information.
Iterates over parameters selected for export and loads
additional data from the referenced files.
:param heat: tht client
:type heat: Client
:param working_dir: Working dir for the deployment
:type working_dir: string
:param stack: stack name for password generator
:type stack: string
:params should_filter:
@ -93,7 +98,6 @@ def export_stack(heat, stack, should_filter=False,
"""
data = {}
heat_stack = oooutils.get_stack(heat, stack)
for export_key, export_param in constants.EXPORT_DATA.items():
param = export_param["parameter"]
@ -106,8 +110,8 @@ def export_stack(heat, stack, should_filter=False,
export_data = oooutils.get_parameter_file(file)
else:
# get stack data
export_data = oooutils.get_stack_output_item(
heat_stack, export_key)
export_data = oooutils.get_stack_saved_output_item(
export_key, working_dir)
if export_data:
# When we export information from a cell controller stack
@ -121,14 +125,21 @@ def export_stack(heat, stack, should_filter=False,
data[param] = export_data
else:
raise RuntimeError(
"No data returned to export %s from." % param)
LOG.warning("No data returned to export %s from." % param)
# Check if AuthCloudName is in the stack environment, and if so add it to
# the export data. Otherwise set it to the exported stack's name.
auth_cloud_name = heat_stack.environment().get(
'parameter_defaults').get(
'AuthCloudName', None)
auth_cloud_name = oooutils.get_stack_saved_output_item(
'AuthCloudName', working_dir)
if auth_cloud_name:
data['AuthCloudName'] = auth_cloud_name
else:
data['AuthCloudName'] = stack
# Check if AuthCloudName is in the stack environment, and if so add it to
# the export data. Otherwise set it to the exported stack's name.
auth_cloud_name = oooutils.get_stack_saved_output_item(
'AuthCloudName', working_dir)
if auth_cloud_name:
data['AuthCloudName'] = auth_cloud_name
else:
@ -213,11 +224,11 @@ def export_ceph(stack, cephx,
return data
def export_overcloud(heat, stack, excludes, should_filter,
def export_overcloud(working_dir, stack, excludes, should_filter,
config_download_dir):
data = export_passwords(heat, stack, excludes)
data = export_passwords(working_dir, stack, excludes)
data.update(export_stack(
heat, stack, should_filter, config_download_dir))
working_dir, stack, should_filter, config_download_dir))
# do not add extra host entries for VIPs for stacks deployed off that
# exported data, since it already contains those entries
data.update({'AddVipsToEtcHosts': False})

View File

@ -12,13 +12,13 @@
# License for the specific language governing permissions and limitations
# under the License.
#
from json.decoder import JSONDecodeError
import os
from unittest import mock
from unittest import TestCase
from tripleoclient import export
from tripleoclient import utils
class TestExport(TestCase):
@ -28,23 +28,8 @@ class TestExport(TestCase):
self.unlink_patch.start()
self.mock_log = mock.Mock('logging.getLogger')
outputs = [
{'output_key': 'EndpointMap',
'output_value': dict(em_key='em_value')},
{'output_key': 'HostsEntry',
'output_value': 'hosts entry'},
{'output_key': 'GlobalConfig',
'output_value': dict(gc_key='gc_value')},
]
self.mock_stack = mock.Mock()
self.mock_stack.to_dict.return_value = dict(outputs=outputs)
self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}')
mock_environment = mock.Mock()
self.mock_stack.environment = mock_environment
mock_environment.return_value = dict(
parameter_defaults=dict())
ceph_inv = {
'DistributedComputeHCI': {
'hosts': {
@ -80,40 +65,23 @@ class TestExport(TestCase):
}
self.mock_open_ceph_all = mock.mock_open(read_data=str(ceph_all))
def _get_stack_saved_output_item(self, output_key, working_dir):
outputs = {
'EndpointMap': dict(em_key='em_value'),
'HostsEntry': 'hosts entry',
'GlobalConfig': dict(gc_key='gc_value'),
'AuthCloudName': 'central',
}
return outputs[output_key]
@mock.patch('tripleoclient.utils.get_stack_saved_output_item')
@mock.patch('tripleoclient.utils.os.path.exists',
autospec=True, reutrn_value=True)
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack(self, mock_get_stack, mock_exists):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
def test_export_stack(self, mock_exists, mock_output_item):
mock_output_item.side_effect = self._get_stack_saved_output_item
working_dir = utils.get_default_working_dir('overcloud')
with mock.patch('tripleoclient.utils.open', self.mock_open):
data = export.export_stack(heat, "overcloud")
expected = \
{'AllNodesExtraMapData': {u'an_key': u'an_value'},
'AuthCloudName': 'overcloud',
'EndpointMapOverride': {'em_key': 'em_value'},
'ExtraHostFileEntries': 'hosts entry',
'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
self.assertEqual(expected, data)
self.mock_open.assert_called_once_with(
os.path.join(
os.environ.get('HOME'),
'config-download/overcloud/group_vars/overcloud.json'),
'r')
@mock.patch('tripleoclient.utils.os.path.exists',
autospec=True, reutrn_value=True)
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack_auth_cloud_name_set(
self, mock_get_stack, mock_exists):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
self.mock_stack.environment.return_value['parameter_defaults'] = (
dict(AuthCloudName='central'))
with mock.patch('tripleoclient.utils.open', self.mock_open):
data = export.export_stack(heat, "overcloud")
data = export.export_stack(working_dir, "overcloud")
expected = \
{'AllNodesExtraMapData': {u'an_key': u'an_value'},
@ -129,20 +97,45 @@ class TestExport(TestCase):
'config-download/overcloud/group_vars/overcloud.json'),
'r')
@mock.patch('tripleoclient.utils.get_stack_saved_output_item')
@mock.patch('tripleoclient.utils.os.path.exists',
autospec=True, reutrn_value=True)
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack_should_filter(self, mock_get_stack, mock_exists):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
def test_export_stack_auth_cloud_name_set(
self, mock_exists, mock_output_item):
mock_output_item.side_effect = self._get_stack_saved_output_item
working_dir = utils.get_default_working_dir('overcloud')
with mock.patch('tripleoclient.utils.open', self.mock_open):
data = export.export_stack(working_dir, "overcloud")
expected = \
{'AllNodesExtraMapData': {u'an_key': u'an_value'},
'AuthCloudName': 'central',
'EndpointMapOverride': {'em_key': 'em_value'},
'ExtraHostFileEntries': 'hosts entry',
'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
self.assertEqual(expected, data)
self.mock_open.assert_called_once_with(
os.path.join(
os.environ.get('HOME'),
'config-download/overcloud/group_vars/overcloud.json'),
'r')
@mock.patch('tripleoclient.utils.get_stack_saved_output_item')
@mock.patch('tripleoclient.utils.os.path.exists',
autospec=True, reutrn_value=True)
def test_export_stack_should_filter(self, mock_exists, mock_stack_output):
working_dir = utils.get_default_working_dir('overcloud')
mock_stack_output.side_effect = self._get_stack_saved_output_item
self.mock_open = mock.mock_open(
read_data='{"an_key":"an_value","ovn_dbs_vip":"vip"}')
with mock.patch('builtins.open', self.mock_open):
data = export.export_stack(heat, "overcloud", should_filter=True)
data = export.export_stack(
working_dir, "overcloud", should_filter=True)
expected = \
{'AllNodesExtraMapData': {u'ovn_dbs_vip': u'vip'},
'AuthCloudName': 'overcloud',
'AuthCloudName': 'central',
'EndpointMapOverride': {'em_key': 'em_value'},
'ExtraHostFileEntries': 'hosts entry',
'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
@ -156,51 +149,17 @@ class TestExport(TestCase):
@mock.patch('tripleoclient.utils.os.path.exists',
autospec=True, reutrn_value=True)
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack_cd_dir(self, mock_get_stack, mock_exists):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
def test_export_stack_cd_dir(self, mock_exists):
working_dir = utils.get_default_working_dir('overcloud')
with mock.patch('tripleoclient.utils.open', self.mock_open):
export.export_stack(heat, "overcloud",
export.export_stack(working_dir, "overcloud",
config_download_dir='/foo')
self.mock_open.assert_called_once_with(
self.mock_open.assert_called_with(
'/foo/overcloud/group_vars/overcloud.json', 'r')
@mock.patch('tripleoclient.utils.os.path.exists',
autospec=True, reutrn_value=True)
@mock.patch('tripleoclient.utils.get_stack')
def test_export_stack_stack_name(self, mock_get_stack, mock_exists):
heat = mock.Mock()
mock_get_stack.return_value = self.mock_stack
with mock.patch('tripleoclient.utils.open', self.mock_open):
export.export_stack(heat, "control")
mock_get_stack.assert_called_once_with(heat, 'control')
@mock.patch('tripleoclient.utils.LOG.error', autospec=True)
@mock.patch('tripleoclient.utils.json.load', autospec=True,
side_effect=JSONDecodeError)
@mock.patch('tripleoclient.utils.open')
@mock.patch('tripleoclient.utils.os.path.exists', autospec=True,
return_value=True)
@mock.patch('tripleoclient.utils.get_stack', autospec=True)
def test_export_stack_decode_error(self, mock_get_stack, mock_exists,
mock_open, mock_json_load, mock_log):
heat = mock.MagicMock()
mock_get_stack.return_value = self.mock_stack
self.assertRaises(
RuntimeError, export.export_stack, heat, "overcloud")
mock_open.assert_called_once_with(
os.path.join(
os.environ.get('HOME'),
'config-download/overcloud/group_vars/overcloud.json'),
'r')
@mock.patch('tripleoclient.export.LOG')
@mock.patch('tripleo_common.utils.plan.generate_passwords')
def test_export_passwords(self, mock_gen_pass, mock_log):
heat = mock.Mock()
mock_passwords = {
'AdminPassword': 'A',
'RpcPassword': 'B',
@ -211,16 +170,20 @@ class TestExport(TestCase):
mock_gen_pass.return_value = mock_passwords
expected_password_export = mock_passwords.copy()
data = export.export_passwords(heat, 'overcloud', False)
working_dir = utils.get_default_working_dir('overcloud')
with mock.patch('builtins.open', mock.mock_open()):
data = export.export_passwords(working_dir, 'overcloud', False)
self.assertEqual(
expected_password_export,
data)
@mock.patch('tripleoclient.utils.get_stack_saved_output_item')
@mock.patch('tripleoclient.export.LOG')
@mock.patch('tripleo_common.utils.plan.generate_passwords')
def test_export_passwords_excludes(self, mock_gen_pass, mock_log):
heat = mock.Mock()
def test_export_passwords_excludes(self, mock_gen_pass, mock_log,
mock_output_item):
mock_output_item.side_effect = self._get_stack_saved_output_item
mock_passwords = {
'AdminPassword': 'A',
'RpcPassword': 'B',
@ -234,7 +197,9 @@ class TestExport(TestCase):
'AdminPassword': 'A',
'RpcPassword': 'B'}
data = export.export_passwords(heat, 'overcloud')
working_dir = utils.get_default_working_dir('overcloud')
with mock.patch('builtins.open', mock.mock_open()):
data = export.export_passwords(working_dir, 'overcloud')
self.assertEqual(expected_password_export, data)

View File

@ -15,9 +15,9 @@ import os
from unittest import mock
from keystoneauth1.exceptions.catalog import EndpointNotFound
from osc_lib.tests import utils
from tripleoclient import utils as ooo_utils
from tripleoclient.v1 import overcloud_export
@ -50,14 +50,14 @@ class TestOvercloudExport(utils.TestCommand):
with mock.patch('builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('overcloud'),
'overcloud', True)
path = os.path.join(os.environ.get('HOME'),
'overcloud-deploy',
'overcloud',
'config-download')
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('overcloud'),
'overcloud',
False,
path)
@ -82,14 +82,14 @@ class TestOvercloudExport(utils.TestCommand):
with mock.patch('builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('foo'),
'foo', True)
path = os.path.join(os.environ.get('HOME'),
'overcloud-deploy',
'foo',
'config-download')
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('foo'),
'foo',
False,
path)
@ -110,10 +110,10 @@ class TestOvercloudExport(utils.TestCommand):
with mock.patch('builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('foo'),
'foo', True)
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('foo'),
'foo',
False,
'/tmp/bar')
@ -136,28 +136,28 @@ class TestOvercloudExport(utils.TestCommand):
with mock.patch('builtins.open', self.mock_open):
self.cmd.take_action(parsed_args)
mock_export_passwords.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('foo'),
'foo', False)
mock_export_stack.assert_called_once_with(
self.app.client_manager.orchestration,
ooo_utils.get_default_working_dir('foo'),
'foo',
False,
'/tmp/bar')
@mock.patch('tripleo_common.utils.plan.generate_passwords')
@mock.patch('shutil.copy')
@mock.patch('os.path.exists')
@mock.patch('tripleoclient.utils.get_default_working_dir')
def test_export_ephemeral_heat(self, mock_working_dir, mock_exists,
mock_copy):
mock_copy, mock_passwords):
argslist = ['--force-overwrite']
verifylist = [('force_overwrite', True)]
parsed_args = self.check_parser(self.cmd, argslist, verifylist)
mock_exists.return_value = True
mock_working_dir.return_value = 'wd'
heat = self.app.client_manager.orchestration
heat.stacks.client.session.get_endpoint.side_effect = EndpointNotFound
with mock.patch('builtins.open', self.mock_open):
mock_open = mock.mock_open(read_data='{}')
mock_passwords.return_value = dict()
with mock.patch('builtins.open', mock_open):
self.cmd.take_action(parsed_args)
mock_working_dir.assert_called()
mock_copy.assert_called_with(
'wd/overcloud-export.yaml', 'overcloud-export.yaml')
mock_passwords.assert_called()

View File

@ -1327,7 +1327,7 @@ class DeployOvercloud(command.Command):
parsed_args.config_download):
# Create overcloud export
data = export.export_overcloud(
self.orchestration_client,
self.working_dir,
parsed_args.stack, True, False,
config_download_dir)
export_file = os.path.join(

View File

@ -13,13 +13,9 @@
from datetime import datetime
import logging
import os.path
import shutil
import sys
import yaml
from keystoneauth1.exceptions.catalog import EndpointNotFound
from osc_lib.i18n import _
from osc_lib import utils as osc_utils
from tripleoclient import command
from tripleoclient import export
@ -39,10 +35,8 @@ class ExportOvercloud(command.Command):
metavar='<stack>',
help=_('Name of the environment main Heat stack '
'to export information from. '
'(default=Env: OVERCLOUD_STACK_NAME)'),
default=osc_utils.env(
'OVERCLOUD_STACK_NAME',
default='overcloud'))
'(default=overcloud)'),
default='overcloud')
parser.add_argument('--output-file', '-o', metavar='<output file>',
help=_('Name of the output file for the stack '
'data export. It will default to '
@ -50,6 +44,12 @@ class ExportOvercloud(command.Command):
parser.add_argument('--force-overwrite', '-f', action='store_true',
default=False,
help=_('Overwrite output file if it exists.'))
parser.add_argument(
'--working-dir',
action='store',
help=_('The working directory for the deployment where all '
'input, output, and generated files are stored.\n'
'Defaults to "$HOME/overcloud-deploy/<stack>"'))
parser.add_argument('--config-download-dir',
action='store',
help=_('Directory to search for config-download '
@ -76,9 +76,10 @@ class ExportOvercloud(command.Command):
self.now,
parsed_args)
if os.path.exists(output_file) and not parsed_args.force_overwrite:
raise Exception(
"File '%s' already exists, not exporting." % output_file)
if not parsed_args.working_dir:
working_dir = utils.get_default_working_dir(stack)
else:
working_dir = parsed_args.working_dir
if not parsed_args.config_download_dir:
config_download_dir = os.path.join(os.environ.get('HOME'),
@ -88,38 +89,19 @@ class ExportOvercloud(command.Command):
else:
config_download_dir = parsed_args.config_download_dir
# prepare clients to access the environment
clients = self.app.client_manager
try:
heat = clients.orchestration
heat.stacks.client.session.get_endpoint(
service_type='orchestration')
except EndpointNotFound:
self.log.warning(
"Heat endpoint not found. When using ephemeral Heat, "
"the export file exists in the stack working directory "
"as $HOME/overlcoud-deploy/<stack>/<stack>-export.yaml. "
"(default). The existing export file will be copied "
"to {}".format(output_file))
export_file_path = os.path.join(
utils.get_default_working_dir(parsed_args.stack),
'{}-export.yaml'.format(parsed_args.stack))
if os.path.exists(export_file_path):
print(
"Export file found at {}, copying to {}.".format(
export_file_path, output_file))
shutil.copy(export_file_path, output_file)
else:
print("Export file not found at {}.".format(
export_file_path))
sys.exit(1)
return
export_file_path = os.path.join(
working_dir,
'{}-export.yaml'.format(parsed_args.stack))
if (os.path.exists(export_file_path) and
not parsed_args.force_overwrite):
raise Exception(
"File '%s' already exists, not exporting." % export_file_path)
data = export.export_overcloud(
heat, stack, excludes=not parsed_args.no_password_excludes,
working_dir, stack, excludes=not parsed_args.no_password_excludes,
should_filter=False, config_download_dir=config_download_dir)
# write the exported data
with open(output_file, 'w') as f:
with open(export_file_path, 'w') as f:
yaml.safe_dump(data, f, default_flow_style=False)
print("Stack information exported to %s." % output_file)