Add openstack overcloud export

Add a new command to export the data from a control plane stack for
multi-stack. The created export file can be used as input into a compute
stack for the multi-stack feature.

Also refactors the overcloud cell export command to use the same common
code.

Closes-Bug: #1850636
Change-Id: I6e145d7a54dcd306c4633ebd4d7471a19a967a86

releasenotes/notes/openstack-overcloud-export-293c8f0f6ab13e91.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - A new command "openstack overcloud export" is added. The command is used to
+    export the data from a control stack for use in a compute stack for the
+    multi-stack feature.

setup.cfg

@@ -62,6 +62,7 @@ openstack.tripleoclient.v1 =
     overcloud_delete = tripleoclient.v1.overcloud_delete:DeleteOvercloud
     overcloud_credentials = tripleoclient.v1.overcloud_credentials:OvercloudCredentials
     overcloud_deploy = tripleoclient.v1.overcloud_deploy:DeployOvercloud
+    overcloud_export = tripleoclient.v1.overcloud_export:ExportOvercloud
     overcloud_failures = tripleoclient.v1.overcloud_deploy:GetDeploymentFailures
     overcloud_status = tripleoclient.v1.overcloud_deploy:GetDeploymentStatus
     overcloud_image_build = tripleoclient.v1.overcloud_image:BuildOvercloudImage

tripleoclient/constants.py

@@ -135,3 +135,7 @@ CLOUDS_YAML_DIR = os.path.join('.config', 'openstack')
 UNUSED_PARAMETER_EXCLUDES_RE = ['^(Docker|Container).*Image$',
                                 '^SwiftFetchDir(Get|Put)Tempurl$',
                                 '^PythonInterpreter$']
+
+EXPORT_PASSWORD_EXCLUDE_PATTERNS = [
+    'ceph.*'
+]

tripleoclient/export.py

@@ -0,0 +1,122 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+import json
+import logging
+import os
+import re
+import sys
+import yaml
+
+from osc_lib.i18n import _
+
+from tripleoclient import constants
+from tripleoclient import utils as oooutils
+
+
+LOG = logging.getLogger(__name__ + ".utils")
+
+
+def export_passwords(swift, stack, excludes=True):
+    # Export the passwords from swift
+    obj = 'plan-environment.yaml'
+    container = stack
+    try:
+        resp_headers, content = swift.get_object(container, obj)
+    except Exception as e:
+        LOG.error("An error happened while exporting the password "
+                  "file from swift: %s", str(e))
+        sys.exit(1)
+
+    data = yaml.load(content)["passwords"]
+    if excludes:
+        excluded_passwords = []
+        for k in data:
+            for pattern in constants.EXPORT_PASSWORD_EXCLUDE_PATTERNS:
+                if re.match(pattern, k, re.I):
+                    excluded_passwords.append(k)
+        [data.pop(e) for e in excluded_passwords]
+    return data
+
+
+def export_stack(heat, stack, should_filter=False,
+                 config_download_dir='/var/lib/mistral/overcloud'):
+
+    # data to export
+    # parameter: Parameter to be exported
+    # file:   IF file specified it is taken as source instead of heat
+    #         output.File is relative to <config-download-dir>/stack.
+    # filter: in case only specific settings should be
+    #         exported from parameter data.
+    export_data = {
+        "EndpointMap": {
+            "parameter": "EndpointMapOverride",
+        },
+        "HostsEntry": {
+            "parameter": "ExtraHostFileEntries",
+        },
+        "GlobalConfig": {
+            "parameter": "GlobalConfigExtraMapData",
+        },
+        "AllNodesConfig": {
+            "file": "group_vars/overcloud.json",
+            "parameter": "AllNodesExtraMapData",
+            "filter": ["oslo_messaging_notify_short_bootstrap_node_name",
+                       "oslo_messaging_notify_node_names",
+                       "oslo_messaging_rpc_node_names",
+                       "memcached_node_ips",
+                       "ovn_dbs_vip",
+                       "redis_vip"]},
+    }
+
+    data = {}
+    heat_stack = oooutils.get_stack(heat, stack)
+
+    for export_key, export_param in export_data.items():
+        param = export_param["parameter"]
+        if "file" in export_param:
+            # get file data
+            file = os.path.join(config_download_dir,
+                                export_param["file"])
+            with open(file, 'r') as ff:
+                try:
+                    export_data = json.load(ff)
+                except Exception as e:
+                    LOG.error(
+                        _('Could not read file %s') % file)
+                    LOG.error(e)
+
+        else:
+            # get stack data
+            export_data = oooutils.get_stack_output_item(
+                            heat_stack, export_key)
+
+        if export_data:
+            # When we export information from a cell controller stack
+            # we don't want to filter.
+            if "filter" in export_param and should_filter:
+                for filter_key in export_param["filter"]:
+                    if filter_key in export_data:
+                        element = {filter_key: export_data[filter_key]}
+                        data.setdefault(param, {}).update(element)
+            else:
+                data[param] = export_data
+
+        else:
+            raise Exception(
+                "No data returned to export %s from." % param)
+
+    return data

tripleoclient/tests/test_export.py

@@ -0,0 +1,139 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+#
+
+
+from io import StringIO
+import mock
+import six
+from unittest import TestCase
+import yaml
+
+from tripleoclient import export
+
+
+class TestExport(TestCase):
+    def setUp(self):
+        self.unlink_patch = mock.patch('os.unlink')
+        self.addCleanup(self.unlink_patch.stop)
+        self.unlink_patch.start()
+        self.mock_log = mock.Mock('logging.getLogger')
+
+        outputs = [
+            {'output_key': 'EndpointMap',
+             'output_value': dict(em_key='em_value')},
+            {'output_key': 'HostsEntry',
+             'output_value': 'hosts entry'},
+            {'output_key': 'GlobalConfig',
+             'output_value': dict(gc_key='gc_value')},
+        ]
+        self.mock_stack = mock.Mock()
+        self.mock_stack.to_dict.return_value = dict(outputs=outputs)
+        self.mock_open = mock.mock_open(read_data='{"an_key":"an_value"}')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            data = export.export_stack(heat, "overcloud")
+
+        expected = \
+            {'AllNodesExtraMapData': {u'an_key': u'an_value'},
+             'EndpointMapOverride': {'em_key': 'em_value'},
+             'ExtraHostFileEntries': 'hosts entry',
+             'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
+
+        self.assertEqual(expected, data)
+        self.mock_open.assert_called_once_with(
+            '/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack_should_filter(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        self.mock_open = mock.mock_open(
+            read_data='{"an_key":"an_value","ovn_dbs_vip":"vip"}')
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            data = export.export_stack(heat, "overcloud", should_filter=True)
+
+        expected = \
+            {'AllNodesExtraMapData': {u'ovn_dbs_vip': u'vip'},
+             'EndpointMapOverride': {'em_key': 'em_value'},
+             'ExtraHostFileEntries': 'hosts entry',
+             'GlobalConfigExtraMapData': {'gc_key': 'gc_value'}}
+
+        self.assertEqual(expected, data)
+        self.mock_open.assert_called_once_with(
+            '/var/lib/mistral/overcloud/group_vars/overcloud.json', 'r')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack_cd_dir(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            export.export_stack(heat, "overcloud",
+                                config_download_dir='/foo/overcloud')
+        self.mock_open.assert_called_once_with(
+            '/foo/overcloud/group_vars/overcloud.json', 'r')
+
+    @mock.patch('tripleoclient.utils.get_stack')
+    def test_export_stack_stack_name(self, mock_get_stack):
+        heat = mock.Mock()
+        mock_get_stack.return_value = self.mock_stack
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            export.export_stack(heat, "control")
+        mock_get_stack.assert_called_once_with(heat, 'control')
+
+    def test_export_passwords(self):
+        swift = mock.Mock()
+        mock_passwords = {
+            'passwords': {
+                'a': 'A',
+                'b': 'B'
+            }
+        }
+        sio = StringIO()
+        sio.write(six.text_type(yaml.dump(mock_passwords)))
+        sio.seek(0)
+        swift.get_object.return_value = ("", sio)
+        data = export.export_passwords(swift, 'overcloud')
+
+        swift.get_object.assert_called_once_with(
+            'overcloud', 'plan-environment.yaml')
+
+        self.assertEqual(mock_passwords['passwords'], data)
+
+    def test_export_passwords_excludes(self):
+        swift = mock.Mock()
+        mock_passwords = {
+            'passwords': {
+                'a': 'A',
+                'b': 'B',
+                'Cephkey': 'cephkey',
+                'cephkey': 'cephkey',
+                'CEPH': 'cephkey'
+            }
+        }
+        sio = StringIO()
+        sio.write(six.text_type(yaml.dump(mock_passwords)))
+        sio.seek(0)
+        swift.get_object.return_value = ("", sio)
+        data = export.export_passwords(swift, 'overcloud')
+
+        mock_passwords['passwords'].pop('Cephkey')
+        mock_passwords['passwords'].pop('cephkey')
+        mock_passwords['passwords'].pop('CEPH')
+
+        self.assertEqual(mock_passwords['passwords'], data)

tripleoclient/tests/v1/test_overcloud_export.py

@@ -0,0 +1,132 @@
+#   Copyright 2019 Red Hat, Inc.
+#
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+import mock
+
+from osc_lib.tests import utils
+
+from tripleoclient.v1 import overcloud_export
+
+
+class TestOvercloudExport(utils.TestCommand):
+
+    def setUp(self):
+        super(TestOvercloudExport, self).setUp()
+
+        self.cmd = overcloud_export.ExportOvercloud(self.app, None)
+        self.app.client_manager.orchestration = mock.Mock()
+        self.tripleoclient = mock.Mock()
+        self.app.client_manager.tripleoclient = self.tripleoclient
+        self.app.client_manager.tripleoclient.object_store = mock.Mock()
+        self.mock_open = mock.mock_open()
+
+    @mock.patch('os.path.exists')
+    @mock.patch('yaml.safe_dump')
+    @mock.patch('tripleoclient.export.export_stack')
+    @mock.patch('tripleoclient.export.export_passwords')
+    def test_export(self, mock_export_passwords,
+                    mock_export_stack,
+                    mock_safe_dump, mock_exists):
+        argslist = []
+        verifylist = []
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        mock_exists.return_value = False
+        mock_export_passwords.return_value = {'key': 'value'}
+        mock_export_stack.return_value = {'key0': 'value0'}
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            self.cmd.take_action(parsed_args)
+        mock_export_passwords.assert_called_once_with(
+            self.app.client_manager.tripleoclient.object_store,
+            'overcloud', True)
+        mock_export_stack.assert_called_once_with(
+            self.app.client_manager.orchestration,
+            'overcloud',
+            False,
+            '/var/lib/mistral/overcloud')
+        self.assertEqual(
+            {'parameter_defaults': {'key': 'value',
+                                    'key0': 'value0'}},
+            mock_safe_dump.call_args[0][0])
+
+    @mock.patch('os.path.exists')
+    @mock.patch('yaml.safe_dump')
+    @mock.patch('tripleoclient.export.export_stack')
+    @mock.patch('tripleoclient.export.export_passwords')
+    def test_export_stack_name(self, mock_export_passwords,
+                               mock_export_stack,
+                               mock_safe_dump, mock_exists):
+        argslist = ['--stack', 'foo']
+        verifylist = [('stack', 'foo')]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        mock_exists.return_value = False
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            self.cmd.take_action(parsed_args)
+        mock_export_passwords.assert_called_once_with(
+            self.app.client_manager.tripleoclient.object_store,
+            'foo', True)
+        mock_export_stack.assert_called_once_with(
+            self.app.client_manager.orchestration,
+            'foo',
+            False,
+            '/var/lib/mistral/foo')
+
+    @mock.patch('os.path.exists')
+    @mock.patch('yaml.safe_dump')
+    @mock.patch('tripleoclient.export.export_stack')
+    @mock.patch('tripleoclient.export.export_passwords')
+    def test_export_stack_name_and_dir(self, mock_export_passwords,
+                                       mock_export_stack,
+                                       mock_safe_dump, mock_exists):
+        argslist = ['--stack', 'foo',
+                    '--config-download-dir', '/tmp/bar']
+        verifylist = [('stack', 'foo'),
+                      ('config_download_dir', '/tmp/bar')]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        mock_exists.return_value = False
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            self.cmd.take_action(parsed_args)
+        mock_export_passwords.assert_called_once_with(
+            self.app.client_manager.tripleoclient.object_store,
+            'foo', True)
+        mock_export_stack.assert_called_once_with(
+            self.app.client_manager.orchestration,
+            'foo',
+            False,
+            '/tmp/bar')
+
+    @mock.patch('os.path.exists')
+    @mock.patch('yaml.safe_dump')
+    @mock.patch('tripleoclient.export.export_stack')
+    @mock.patch('tripleoclient.export.export_passwords')
+    def test_export_no_excludes(self, mock_export_passwords,
+                                mock_export_stack,
+                                mock_safe_dump, mock_exists):
+        argslist = ['--stack', 'foo',
+                    '--config-download-dir', '/tmp/bar',
+                    '--no-password-excludes']
+        verifylist = [('stack', 'foo'),
+                      ('config_download_dir', '/tmp/bar'),
+                      ('no_password_excludes', True)]
+        parsed_args = self.check_parser(self.cmd, argslist, verifylist)
+        mock_exists.return_value = False
+        with mock.patch('six.moves.builtins.open', self.mock_open):
+            self.cmd.take_action(parsed_args)
+        mock_export_passwords.assert_called_once_with(
+            self.app.client_manager.tripleoclient.object_store,
+            'foo', False)
+        mock_export_stack.assert_called_once_with(
+            self.app.client_manager.orchestration,
+            'foo',
+            False,
+            '/tmp/bar')

tripleoclient/v1/overcloud_cell.py

@@ -13,10 +13,8 @@
 from __future__ import print_function
 
 from datetime import datetime
-import json
 import logging
 import os.path
-import sys
 import yaml
 
 from osc_lib.i18n import _
@@ -24,7 +22,8 @@ from osc_lib import utils
 
 from tripleoclient import command
 from tripleoclient import exceptions
-from tripleoclient import utils as oooutils
+from tripleoclient import export
+
 
 MISTRAL_VAR = os.environ.get('MISTRAL_VAR',
                              "/var/lib/mistral/")
@@ -83,95 +82,22 @@ class ExportCell(command.Command):
         clients = self.app.client_manager
         swift_client = clients.tripleoclient.object_store
 
-        # data to export
+        data = export.export_passwords(swift_client, control_plane_stack)
-        # parameter: Parameter to be exported
-        # file:   IF file specified it is taken as source instead of heat
-        #         output.File is relative to MISTRAL_VAR/stack_to_export.
-        # filter: in case only specific settings should be
-        #         exported from parameter data.
-        export_data = {
-            "EndpointMap": {
-                "parameter": "EndpointMapOverride",
-            },
-            "HostsEntry": {
-                "parameter": "ExtraHostFileEntries",
-            },
-            "GlobalConfig": {
-                "parameter": "GlobalConfigExtraMapData",
-            },
-            "AllNodesConfig": {
-                "file": "/group_vars/overcloud.json",
-                "parameter": "GlobalConfigExtraMapData",
-                "filter": ["oslo_messaging_notify_short_bootstrap_node_name",
-                           "oslo_messaging_notify_node_names",
-                           "oslo_messaging_rpc_node_names",
-                           "memcached_node_ips",
-                           "ovn_dbs_vip",
-                           "redis_vip"]},
-        }
-
-        # export the data from swift and heat
-        data_real = {}
-
-        # Export the passwords from swift
-        obj = 'plan-environment.yaml'
-        container = control_plane_stack
-        try:
-            resp_headers, content = swift_client.get_object(container, obj)
-        except Exception as e:
-            self.log.error("An error happened while exporting the password "
-                           "file from swift: %s", str(e))
-            sys.exit(1)
-
-        data_real = {'parameter_defaults': yaml.load(content)["passwords"]}
 
         stack_to_export = control_plane_stack
+        should_filter = True
         if cell_stack:
             stack_to_export = cell_stack
+            should_filter = False
 
-        stack = oooutils.get_stack(clients.orchestration, stack_to_export)
+        config_download_dir = os.path.join(MISTRAL_VAR, stack_to_export)
-
+        data['parameter_defaults'].update(export.export_stack(
-        for export_key, export_param in export_data.items():
+            clients.orchestration, stack_to_export, should_filter,
-            data = None
+            config_download_dir))
-            if "file" in export_param:
-                # get stack data
-                file = MISTRAL_VAR + stack_to_export + export_param["file"]
-                with open(file, 'r') as ff:
-                    try:
-                        data = json.load(ff)
-                    except Exception:
-                        self.log.error(
-                            _('Could not read file %s') % file)
-            else:
-                # get stack data
-                data = oooutils.get_stack_output_item(stack,
-                                                      export_key)
-
-            param = export_param["parameter"]
-            if data:
-                # do we just want a subset of entries?
-                # When we export information from a cell controller stack
-                # we don't want to filter.
-                if "filter" in export_param and not cell_stack:
-                    for x in export_param["filter"]:
-                        element = {x: data[x]}
-                        if param not in data_real["parameter_defaults"]:
-                            data_real["parameter_defaults"][param] = element
-                        else:
-                            data_real["parameter_defaults"][param].update(
-                                element)
-                else:
-                    if param not in data_real["parameter_defaults"]:
-                        data_real["parameter_defaults"][param] = data
-                    else:
-                        data_real["parameter_defaults"][param].update(data)
-            else:
-                raise exceptions.CellExportError(
-                    "No data returned to export %s from." % param)
 
         # write the exported data
         with open(output_file, 'w') as f:
-            yaml.safe_dump(data_real, f, default_flow_style=False)
+            yaml.safe_dump(data, f, default_flow_style=False)
 
         print("Cell input information exported to %s." % output_file)
 

tripleoclient/v1/overcloud_export.py

@@ -0,0 +1,100 @@
+#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+#   not use this file except in compliance with the License. You may obtain
+#   a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#   License for the specific language governing permissions and limitations
+#   under the License.
+
+from __future__ import print_function
+
+from datetime import datetime
+import logging
+import os.path
+import yaml
+
+from osc_lib.i18n import _
+from osc_lib import utils
+
+from tripleoclient import command
+from tripleoclient import export
+
+
+class ExportOvercloud(command.Command):
+    """Export stack information used as import of another stack"""
+
+    log = logging.getLogger(__name__ + ".ExportOvercloud")
+    now = datetime.now().strftime('%Y%m%d%H%M%S')
+
+    def get_parser(self, prog_name):
+        parser = super(ExportOvercloud, self).get_parser(prog_name)
+        parser.add_argument('--stack',
+                            dest='stack',
+                            metavar='<stack>',
+                            help=_('Name of the environment main Heat stack '
+                                   'to export information from. '
+                                   '(default=Env: OVERCLOUD_STACK_NAME)'),
+                            default=utils.env('OVERCLOUD_STACK_NAME',
+                                              default='overcloud'))
+        parser.add_argument('--output-file', '-o', metavar='<output file>',
+                            help=_('Name of the output file for the stack '
+                                   'data export. It will default to '
+                                   '"<name>.yaml"'))
+        parser.add_argument('--force-overwrite', '-f', action='store_true',
+                            default=False,
+                            help=_('Overwrite output file if it exists.'))
+        parser.add_argument('--config-download-dir',
+                            action='store',
+                            help=_('Directory to search for config-download '
+                                   'export data. Defaults to '
+                                   '/var/lib/mistral/<stack>'))
+        parser.add_argument('--no-password-excludes',
+                            action='store_true',
+                            dest='no_password_excludes',
+                            help=_('Don''t exclude certain passwords from '
+                                   'the password export. Defaults to False '
+                                   'in that some passwords will be excluded '
+                                   'that are not typically necessary.'))
+
+        return parser
+
+    def take_action(self, parsed_args):
+        self.log.debug("take_action(%s)" % parsed_args)
+
+        stack = parsed_args.stack
+        output_file = parsed_args.output_file or \
+            '%s-export.yaml' % stack
+
+        self.log.info('Running at %s with parameters %s',
+                      self.now,
+                      parsed_args)
+
+        if os.path.exists(output_file) and not parsed_args.force_overwrite:
+            raise Exception(
+                "File '%s' already exists, not exporting." % output_file)
+
+        if not parsed_args.config_download_dir:
+            config_download_dir = os.path.join('/var/lib/mistral',
+                                               parsed_args.stack)
+        else:
+            config_download_dir = parsed_args.config_download_dir
+
+        # prepare clients to access the environment
+        clients = self.app.client_manager
+        swift_client = clients.tripleoclient.object_store
+
+        data = export.export_passwords(swift_client, stack,
+                                       not parsed_args.no_password_excludes)
+        data.update(export.export_stack(
+            clients.orchestration, stack, False, config_download_dir))
+        data = dict(parameter_defaults=data)
+
+        # write the exported data
+        with open(output_file, 'w') as f:
+            yaml.safe_dump(data, f, default_flow_style=False)
+
+        print("Stack information exported to %s." % output_file)