From df4694b74369f3d6d860b5f432e3cac8344226c2 Mon Sep 17 00:00:00 2001 From: James Slagle Date: Fri, 16 Jul 2021 08:16:04 -0400 Subject: [PATCH] Remove Heat API dependency for overcloud export Update the overcloud export command to work without a dependency on Heat API. The export data is updated to create the export data based solely on saved stack output data and the config-download generated overcloud.json so that Heat is not needed to generate the export at all. Change-Id: I0bd4b4a0e33184e0f4d731899138bf4063f3cdac Signed-off-by: James Slagle (cherry picked from commit d7af2f54826b3086af4691753b00173dc7a95a70) --- tripleoclient/export.py | 47 ++++-- tripleoclient/tests/test_export.py | 155 +++++++----------- .../tests/v1/test_overcloud_export.py | 30 ++-- tripleoclient/v1/overcloud_deploy.py | 2 +- tripleoclient/v1/overcloud_export.py | 60 +++---- 5 files changed, 126 insertions(+), 168 deletions(-) diff --git a/tripleoclient/export.py b/tripleoclient/export.py index e93b095c2..be72ec996 100644 --- a/tripleoclient/export.py +++ b/tripleoclient/export.py @@ -29,12 +29,12 @@ from tripleoclient import utils as oooutils LOG = logging.getLogger(__name__ + ".utils") -def export_passwords(heat, stack, excludes=True): +def export_passwords(working_dir, stack, excludes=True): """For each password, check if it's excluded, then check if there's a user defined value from parameter_defaults, and if not use the value from the generated passwords. - :param heat: tht client - :type heat: Client + :param working_dir: Working dir for the deployment + :type working_dir: string :param stack: stack name for password generator :type stack: string :param excludes: filter the passwords or not, defaults to `True` @@ -48,8 +48,13 @@ def export_passwords(heat, stack, excludes=True): if re.match(pattern, password, re.I): return True + passwords_file = os.path.join( + working_dir, + constants.PASSWORDS_ENV_FORMAT.format(stack)) + with open(passwords_file) as f: + passwords_env = yaml.safe_load(f.read()) generated_passwords = plan_utils.generate_passwords( - heat=heat, container=stack) + passwords_env=passwords_env) filtered_passwords = generated_passwords.copy() @@ -61,14 +66,14 @@ def export_passwords(heat, stack, excludes=True): return filtered_passwords -def export_stack(heat, stack, should_filter=False, +def export_stack(working_dir, stack, should_filter=False, config_download_dir=constants.DEFAULT_WORK_DIR): """Export stack information. Iterates over parameters selected for export and loads additional data from the referenced files. - :param heat: tht client - :type heat: Client + :param working_dir: Working dir for the deployment + :type working_dir: string :param stack: stack name for password generator :type stack: string :params should_filter: @@ -93,7 +98,6 @@ def export_stack(heat, stack, should_filter=False, """ data = {} - heat_stack = oooutils.get_stack(heat, stack) for export_key, export_param in constants.EXPORT_DATA.items(): param = export_param["parameter"] @@ -106,8 +110,8 @@ def export_stack(heat, stack, should_filter=False, export_data = oooutils.get_parameter_file(file) else: # get stack data - export_data = oooutils.get_stack_output_item( - heat_stack, export_key) + export_data = oooutils.get_stack_saved_output_item( + export_key, working_dir) if export_data: # When we export information from a cell controller stack @@ -121,14 +125,21 @@ def export_stack(heat, stack, should_filter=False, data[param] = export_data else: - raise RuntimeError( - "No data returned to export %s from." % param) + LOG.warning("No data returned to export %s from." % param) # Check if AuthCloudName is in the stack environment, and if so add it to # the export data. Otherwise set it to the exported stack's name. - auth_cloud_name = heat_stack.environment().get( - 'parameter_defaults').get( - 'AuthCloudName', None) + auth_cloud_name = oooutils.get_stack_saved_output_item( + 'AuthCloudName', working_dir) + if auth_cloud_name: + data['AuthCloudName'] = auth_cloud_name + else: + data['AuthCloudName'] = stack + + # Check if AuthCloudName is in the stack environment, and if so add it to + # the export data. Otherwise set it to the exported stack's name. + auth_cloud_name = oooutils.get_stack_saved_output_item( + 'AuthCloudName', working_dir) if auth_cloud_name: data['AuthCloudName'] = auth_cloud_name else: @@ -213,11 +224,11 @@ def export_ceph(stack, cephx, return data -def export_overcloud(heat, stack, excludes, should_filter, +def export_overcloud(working_dir, stack, excludes, should_filter, config_download_dir): - data = export_passwords(heat, stack, excludes) + data = export_passwords(working_dir, stack, excludes) data.update(export_stack( - heat, stack, should_filter, config_download_dir)) + working_dir, stack, should_filter, config_download_dir)) # do not add extra host entries for VIPs for stacks deployed off that # exported data, since it already contains those entries data.update({'AddVipsToEtcHosts': False}) diff --git a/tripleoclient/tests/test_export.py b/tripleoclient/tests/test_export.py index 8fca2175b..0d49b2d7c 100644 --- a/tripleoclient/tests/test_export.py +++ b/tripleoclient/tests/test_export.py @@ -12,13 +12,13 @@ # License for the specific language governing permissions and limitations # under the License. # -from json.decoder import JSONDecodeError import os import mock from unittest import TestCase from tripleoclient import export +from tripleoclient import utils class TestExport(TestCase): @@ -28,23 +28,8 @@ class TestExport(TestCase): self.unlink_patch.start() self.mock_log = mock.Mock('logging.getLogger') - outputs = [ - {'output_key': 'EndpointMap', - 'output_value': dict(em_key='em_value')}, - {'output_key': 'HostsEntry', - 'output_value': 'hosts entry'}, - {'output_key': 'GlobalConfig', - 'output_value': dict(gc_key='gc_value')}, - ] - self.mock_stack = mock.Mock() - self.mock_stack.to_dict.return_value = dict(outputs=outputs) self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}') - mock_environment = mock.Mock() - self.mock_stack.environment = mock_environment - mock_environment.return_value = dict( - parameter_defaults=dict()) - ceph_inv = { 'DistributedComputeHCI': { 'hosts': { @@ -80,40 +65,23 @@ class TestExport(TestCase): } self.mock_open_ceph_all = mock.mock_open(read_data=str(ceph_all)) + def _get_stack_saved_output_item(self, output_key, working_dir): + outputs = { + 'EndpointMap': dict(em_key='em_value'), + 'HostsEntry': 'hosts entry', + 'GlobalConfig': dict(gc_key='gc_value'), + 'AuthCloudName': 'central', + } + return outputs[output_key] + + @mock.patch('tripleoclient.utils.get_stack_saved_output_item') @mock.patch('tripleoclient.utils.os.path.exists', autospec=True, reutrn_value=True) - @mock.patch('tripleoclient.utils.get_stack') - def test_export_stack(self, mock_get_stack, mock_exists): - heat = mock.Mock() - mock_get_stack.return_value = self.mock_stack + def test_export_stack(self, mock_exists, mock_output_item): + mock_output_item.side_effect = self._get_stack_saved_output_item + working_dir = utils.get_default_working_dir('overcloud') with mock.patch('tripleoclient.utils.open', self.mock_open): - data = export.export_stack(heat, "overcloud") - - expected = \ - {'AllNodesExtraMapData': {u'an_key': u'an_value'}, - 'AuthCloudName': 'overcloud', - 'EndpointMapOverride': {'em_key': 'em_value'}, - 'ExtraHostFileEntries': 'hosts entry', - 'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}} - - self.assertEqual(expected, data) - self.mock_open.assert_called_once_with( - os.path.join( - os.environ.get('HOME'), - 'config-download/overcloud/group_vars/overcloud.json'), - 'r') - - @mock.patch('tripleoclient.utils.os.path.exists', - autospec=True, reutrn_value=True) - @mock.patch('tripleoclient.utils.get_stack') - def test_export_stack_auth_cloud_name_set( - self, mock_get_stack, mock_exists): - heat = mock.Mock() - mock_get_stack.return_value = self.mock_stack - self.mock_stack.environment.return_value['parameter_defaults'] = ( - dict(AuthCloudName='central')) - with mock.patch('tripleoclient.utils.open', self.mock_open): - data = export.export_stack(heat, "overcloud") + data = export.export_stack(working_dir, "overcloud") expected = \ {'AllNodesExtraMapData': {u'an_key': u'an_value'}, @@ -129,20 +97,45 @@ class TestExport(TestCase): 'config-download/overcloud/group_vars/overcloud.json'), 'r') + @mock.patch('tripleoclient.utils.get_stack_saved_output_item') @mock.patch('tripleoclient.utils.os.path.exists', autospec=True, reutrn_value=True) - @mock.patch('tripleoclient.utils.get_stack') - def test_export_stack_should_filter(self, mock_get_stack, mock_exists): - heat = mock.Mock() - mock_get_stack.return_value = self.mock_stack + def test_export_stack_auth_cloud_name_set( + self, mock_exists, mock_output_item): + mock_output_item.side_effect = self._get_stack_saved_output_item + working_dir = utils.get_default_working_dir('overcloud') + with mock.patch('tripleoclient.utils.open', self.mock_open): + data = export.export_stack(working_dir, "overcloud") + + expected = \ + {'AllNodesExtraMapData': {u'an_key': u'an_value'}, + 'AuthCloudName': 'central', + 'EndpointMapOverride': {'em_key': 'em_value'}, + 'ExtraHostFileEntries': 'hosts entry', + 'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}} + + self.assertEqual(expected, data) + self.mock_open.assert_called_once_with( + os.path.join( + os.environ.get('HOME'), + 'config-download/overcloud/group_vars/overcloud.json'), + 'r') + + @mock.patch('tripleoclient.utils.get_stack_saved_output_item') + @mock.patch('tripleoclient.utils.os.path.exists', + autospec=True, reutrn_value=True) + def test_export_stack_should_filter(self, mock_exists, mock_stack_output): + working_dir = utils.get_default_working_dir('overcloud') + mock_stack_output.side_effect = self._get_stack_saved_output_item self.mock_open = mock.mock_open( read_data='{"an_key":"an_value","ovn_dbs_vip":"vip"}') with mock.patch('builtins.open', self.mock_open): - data = export.export_stack(heat, "overcloud", should_filter=True) + data = export.export_stack( + working_dir, "overcloud", should_filter=True) expected = \ {'AllNodesExtraMapData': {u'ovn_dbs_vip': u'vip'}, - 'AuthCloudName': 'overcloud', + 'AuthCloudName': 'central', 'EndpointMapOverride': {'em_key': 'em_value'}, 'ExtraHostFileEntries': 'hosts entry', 'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}} @@ -156,51 +149,17 @@ class TestExport(TestCase): @mock.patch('tripleoclient.utils.os.path.exists', autospec=True, reutrn_value=True) - @mock.patch('tripleoclient.utils.get_stack') - def test_export_stack_cd_dir(self, mock_get_stack, mock_exists): - heat = mock.Mock() - mock_get_stack.return_value = self.mock_stack + def test_export_stack_cd_dir(self, mock_exists): + working_dir = utils.get_default_working_dir('overcloud') with mock.patch('tripleoclient.utils.open', self.mock_open): - export.export_stack(heat, "overcloud", + export.export_stack(working_dir, "overcloud", config_download_dir='/foo') - self.mock_open.assert_called_once_with( + self.mock_open.assert_called_with( '/foo/overcloud/group_vars/overcloud.json', 'r') - @mock.patch('tripleoclient.utils.os.path.exists', - autospec=True, reutrn_value=True) - @mock.patch('tripleoclient.utils.get_stack') - def test_export_stack_stack_name(self, mock_get_stack, mock_exists): - heat = mock.Mock() - mock_get_stack.return_value = self.mock_stack - with mock.patch('tripleoclient.utils.open', self.mock_open): - export.export_stack(heat, "control") - mock_get_stack.assert_called_once_with(heat, 'control') - - @mock.patch('tripleoclient.utils.LOG.error', autospec=True) - @mock.patch('tripleoclient.utils.json.load', autospec=True, - side_effect=JSONDecodeError) - @mock.patch('tripleoclient.utils.open') - @mock.patch('tripleoclient.utils.os.path.exists', autospec=True, - return_value=True) - @mock.patch('tripleoclient.utils.get_stack', autospec=True) - def test_export_stack_decode_error(self, mock_get_stack, mock_exists, - mock_open, mock_json_load, mock_log): - - heat = mock.MagicMock() - mock_get_stack.return_value = self.mock_stack - self.assertRaises( - RuntimeError, export.export_stack, heat, "overcloud") - - mock_open.assert_called_once_with( - os.path.join( - os.environ.get('HOME'), - 'config-download/overcloud/group_vars/overcloud.json'), - 'r') - @mock.patch('tripleoclient.export.LOG') @mock.patch('tripleo_common.utils.plan.generate_passwords') def test_export_passwords(self, mock_gen_pass, mock_log): - heat = mock.Mock() mock_passwords = { 'AdminPassword': 'A', 'RpcPassword': 'B', @@ -211,16 +170,20 @@ class TestExport(TestCase): mock_gen_pass.return_value = mock_passwords expected_password_export = mock_passwords.copy() - data = export.export_passwords(heat, 'overcloud', False) + working_dir = utils.get_default_working_dir('overcloud') + with mock.patch('builtins.open', mock.mock_open()): + data = export.export_passwords(working_dir, 'overcloud', False) self.assertEqual( expected_password_export, data) + @mock.patch('tripleoclient.utils.get_stack_saved_output_item') @mock.patch('tripleoclient.export.LOG') @mock.patch('tripleo_common.utils.plan.generate_passwords') - def test_export_passwords_excludes(self, mock_gen_pass, mock_log): - heat = mock.Mock() + def test_export_passwords_excludes(self, mock_gen_pass, mock_log, + mock_output_item): + mock_output_item.side_effect = self._get_stack_saved_output_item mock_passwords = { 'AdminPassword': 'A', 'RpcPassword': 'B', @@ -234,7 +197,9 @@ class TestExport(TestCase): 'AdminPassword': 'A', 'RpcPassword': 'B'} - data = export.export_passwords(heat, 'overcloud') + working_dir = utils.get_default_working_dir('overcloud') + with mock.patch('builtins.open', mock.mock_open()): + data = export.export_passwords(working_dir, 'overcloud') self.assertEqual(expected_password_export, data) diff --git a/tripleoclient/tests/v1/test_overcloud_export.py b/tripleoclient/tests/v1/test_overcloud_export.py index e6386d020..9e693929b 100644 --- a/tripleoclient/tests/v1/test_overcloud_export.py +++ b/tripleoclient/tests/v1/test_overcloud_export.py @@ -15,9 +15,9 @@ import os import mock -from keystoneauth1.exceptions.catalog import EndpointNotFound from osc_lib.tests import utils +from tripleoclient import utils as ooo_utils from tripleoclient.v1 import overcloud_export @@ -50,14 +50,14 @@ class TestOvercloudExport(utils.TestCommand): with mock.patch('builtins.open', self.mock_open): self.cmd.take_action(parsed_args) mock_export_passwords.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('overcloud'), 'overcloud', True) path = os.path.join(os.environ.get('HOME'), 'overcloud-deploy', 'overcloud', 'config-download') mock_export_stack.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('overcloud'), 'overcloud', False, path) @@ -82,14 +82,14 @@ class TestOvercloudExport(utils.TestCommand): with mock.patch('builtins.open', self.mock_open): self.cmd.take_action(parsed_args) mock_export_passwords.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('foo'), 'foo', True) path = os.path.join(os.environ.get('HOME'), 'overcloud-deploy', 'foo', 'config-download') mock_export_stack.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('foo'), 'foo', False, path) @@ -110,10 +110,10 @@ class TestOvercloudExport(utils.TestCommand): with mock.patch('builtins.open', self.mock_open): self.cmd.take_action(parsed_args) mock_export_passwords.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('foo'), 'foo', True) mock_export_stack.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('foo'), 'foo', False, '/tmp/bar') @@ -136,28 +136,28 @@ class TestOvercloudExport(utils.TestCommand): with mock.patch('builtins.open', self.mock_open): self.cmd.take_action(parsed_args) mock_export_passwords.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('foo'), 'foo', False) mock_export_stack.assert_called_once_with( - self.app.client_manager.orchestration, + ooo_utils.get_default_working_dir('foo'), 'foo', False, '/tmp/bar') + @mock.patch('tripleo_common.utils.plan.generate_passwords') @mock.patch('shutil.copy') @mock.patch('os.path.exists') @mock.patch('tripleoclient.utils.get_default_working_dir') def test_export_ephemeral_heat(self, mock_working_dir, mock_exists, - mock_copy): + mock_copy, mock_passwords): argslist = ['--force-overwrite'] verifylist = [('force_overwrite', True)] parsed_args = self.check_parser(self.cmd, argslist, verifylist) mock_exists.return_value = True mock_working_dir.return_value = 'wd' - heat = self.app.client_manager.orchestration - heat.stacks.client.session.get_endpoint.side_effect = EndpointNotFound - with mock.patch('six.moves.builtins.open', self.mock_open): + mock_open = mock.mock_open(read_data='{}') + mock_passwords.return_value = dict() + with mock.patch('six.moves.builtins.open', mock_open): self.cmd.take_action(parsed_args) mock_working_dir.assert_called() - mock_copy.assert_called_with( - 'wd/overcloud-export.yaml', 'overcloud-export.yaml') + mock_passwords.assert_called() diff --git a/tripleoclient/v1/overcloud_deploy.py b/tripleoclient/v1/overcloud_deploy.py index 0c90d8d50..05d6ae022 100644 --- a/tripleoclient/v1/overcloud_deploy.py +++ b/tripleoclient/v1/overcloud_deploy.py @@ -1392,7 +1392,7 @@ class DeployOvercloud(command.Command): parsed_args.config_download): # Create overcloud export data = export.export_overcloud( - self.orchestration_client, + self.working_dir, parsed_args.stack, True, False, config_download_dir) export_file = os.path.join( diff --git a/tripleoclient/v1/overcloud_export.py b/tripleoclient/v1/overcloud_export.py index 117f0e1ad..6447a5f09 100644 --- a/tripleoclient/v1/overcloud_export.py +++ b/tripleoclient/v1/overcloud_export.py @@ -13,13 +13,9 @@ from datetime import datetime import logging import os.path -import shutil -import sys import yaml -from keystoneauth1.exceptions.catalog import EndpointNotFound from osc_lib.i18n import _ -from osc_lib import utils as osc_utils from tripleoclient import command from tripleoclient import export @@ -39,10 +35,8 @@ class ExportOvercloud(command.Command): metavar='', help=_('Name of the environment main Heat stack ' 'to export information from. ' - '(default=Env: OVERCLOUD_STACK_NAME)'), - default=osc_utils.env( - 'OVERCLOUD_STACK_NAME', - default='overcloud')) + '(default=overcloud)'), + default='overcloud') parser.add_argument('--output-file', '-o', metavar='', help=_('Name of the output file for the stack ' 'data export. It will default to ' @@ -50,6 +44,12 @@ class ExportOvercloud(command.Command): parser.add_argument('--force-overwrite', '-f', action='store_true', default=False, help=_('Overwrite output file if it exists.')) + parser.add_argument( + '--working-dir', + action='store', + help=_('The working directory for the deployment where all ' + 'input, output, and generated files are stored.\n' + 'Defaults to "$HOME/overcloud-deploy/"')) parser.add_argument('--config-download-dir', action='store', help=_('Directory to search for config-download ' @@ -76,9 +76,10 @@ class ExportOvercloud(command.Command): self.now, parsed_args) - if os.path.exists(output_file) and not parsed_args.force_overwrite: - raise Exception( - "File '%s' already exists, not exporting." % output_file) + if not parsed_args.working_dir: + working_dir = utils.get_default_working_dir(stack) + else: + working_dir = parsed_args.working_dir if not parsed_args.config_download_dir: config_download_dir = os.path.join(os.environ.get('HOME'), @@ -88,38 +89,19 @@ class ExportOvercloud(command.Command): else: config_download_dir = parsed_args.config_download_dir - # prepare clients to access the environment - clients = self.app.client_manager - try: - heat = clients.orchestration - heat.stacks.client.session.get_endpoint( - service_type='orchestration') - except EndpointNotFound: - self.log.warning( - "Heat endpoint not found. When using ephemeral Heat, " - "the export file exists in the stack working directory " - "as $HOME/overlcoud-deploy//-export.yaml. " - "(default). The existing export file will be copied " - "to {}".format(output_file)) - export_file_path = os.path.join( - utils.get_default_working_dir(parsed_args.stack), - '{}-export.yaml'.format(parsed_args.stack)) - if os.path.exists(export_file_path): - print( - "Export file found at {}, copying to {}.".format( - export_file_path, output_file)) - shutil.copy(export_file_path, output_file) - else: - print("Export file not found at {}.".format( - export_file_path)) - sys.exit(1) - return + export_file_path = os.path.join( + working_dir, + '{}-export.yaml'.format(parsed_args.stack)) + if (os.path.exists(export_file_path) and + not parsed_args.force_overwrite): + raise Exception( + "File '%s' already exists, not exporting." % export_file_path) data = export.export_overcloud( - heat, stack, excludes=not parsed_args.no_password_excludes, + working_dir, stack, excludes=not parsed_args.no_password_excludes, should_filter=False, config_download_dir=config_download_dir) # write the exported data - with open(output_file, 'w') as f: + with open(export_file_path, 'w') as f: yaml.safe_dump(data, f, default_flow_style=False) print("Stack information exported to %s." % output_file)