overcloud node extract add network info

Extend the command which extract overcloud node
information to also include network information.

The command now requires the --roles-file used
when deploying the overcloud to be passed because
this is the source for these options:
 * default_route_networks
 * networks_skip_config

Partial-Implements: blueprint network-data-v2-ports
Depends-On: https://review.opendev.org/772359
Change-Id: I09c886fe6bada721caac61d25d9a0915cf9aff02

tripleoclient/constants.py

@@ -150,6 +150,7 @@ VALIDATION_GROUPS_INFO = (
         )
 
 # ctlplane network defaults
+CTLPLANE_NET_NAME = 'ctlplane'
 CTLPLANE_CIDR_DEFAULT = '192.168.24.0/24'
 CTLPLANE_DHCP_START_DEFAULT = ['192.168.24.5']
 CTLPLANE_DHCP_END_DEFAULT = ['192.168.24.24']

tripleoclient/tests/v1/overcloud_node/test_overcloud_node.py

@@ -760,18 +760,35 @@ class TestExtractProvisionedNode(test_utils.TestCommand):
         self.baremetal = mock.Mock()
         self.app.client_manager.baremetal = self.baremetal
 
+        self.network = mock.Mock()
+        self.app.client_manager.network = self.network
+
         self.cmd = overcloud_node.ExtractProvisionedNode(self.app, None)
 
         self.extract_file = tempfile.NamedTemporaryFile(
             mode='w', delete=False, suffix='.yaml')
         self.extract_file.close()
+
+        roles_data = [
+            {'name': 'Controller',
+             'default_route_networks': ['External'],
+             'networks_skip_config': ['Tenant']},
+            {'name': 'Compute'}
+        ]
+        self.roles_file = tempfile.NamedTemporaryFile(
+            mode='w', delete=False, suffix='.yaml')
+        self.roles_file.write(yaml.safe_dump(roles_data))
+        self.roles_file.close()
         self.addCleanup(os.unlink, self.extract_file.name)
+        self.addCleanup(os.unlink, self.roles_file.name)
 
     def test_extract(self):
         stack_dict = {
             'parameters': {
                 'ComputeHostnameFormat': '%stackname%-novacompute-%index%',
-                'ControllerHostnameFormat': '%stackname%-controller-%index%'
+                'ControllerHostnameFormat': '%stackname%-controller-%index%',
+                'ComputeNetworkConfigTemplate': 'templates/compute.j2',
+                'ControllerNetworkConfigTemplate': 'templates/controller.j2'
             },
             'outputs': [{
                 'output_key': 'AnsibleHostVarsMap',
@@ -785,6 +802,25 @@ class TestExtractProvisionedNode(test_utils.TestCommand):
                         'overcloud-controller-2'
                     ],
                 }
+            }, {
+                'output_key': 'RoleNetIpMap',
+                'output_value': {
+                    'Compute': {
+                        'ctlplane': ['192.168.26.11'],
+                        'internal_api': ['172.17.1.23'],
+                    },
+                    'Controller': {
+                        'ctlplane': ['192.168.25.21',
+                                     '192.168.25.25',
+                                     '192.168.25.28'],
+                        'external': ['10.0.0.199',
+                                     '10.0.0.197',
+                                     '10.0.0.191'],
+                        'internal_api': ['172.17.0.37',
+                                         '172.17.0.33',
+                                         '172.17.0.39'],
+                    }
+                }
             }]
         }
         stack = mock.Mock()
@@ -809,9 +845,78 @@ class TestExtractProvisionedNode(test_utils.TestCommand):
 
         self.baremetal.node.list.return_value = nodes
 
-        argslist = ['--output', self.extract_file.name, '--yes']
+        networks = [
+            mock.Mock(),  # ctlplane
+            mock.Mock(),  # external
+            mock.Mock(),  # internal_api
+        ]
+        ctlplane_net = networks[0]
+        external_net = networks[1]
+        internal_api_net = networks[2]
+
+        ctlplane_net.id = 'ctlplane_id'
+        ctlplane_net.name = 'ctlplane'
+        ctlplane_net.subnet_ids = ['ctlplane_a_id',
+                                   'ctlplane_b_id']
+        external_net.id = 'external_id'
+        external_net.name = 'external'
+        external_net.subnet_ids = ['external_a_id']
+        internal_api_net.id = 'internal_api_id'
+        internal_api_net.name = 'internal_api'
+        internal_api_net.subnet_ids = ['internal_api_a_id',
+                                       'internal_api_b_id']
+
+        subnets = [
+            mock.Mock(),  # ctlplane_a
+            mock.Mock(),  # ctlplane_b
+            mock.Mock(),  # external_a
+            mock.Mock(),  # internal_api_a
+            mock.Mock(),  # internal_api_b
+        ]
+        ctlplane_a = subnets[0]
+        ctlplane_b = subnets[1]
+        external_a = subnets[2]
+        int_api_a = subnets[3]
+        int_api_b = subnets[4]
+
+        ctlplane_a.id = 'ctlplane_a_id'
+        ctlplane_a.name = 'ctlplane_a'
+        ctlplane_a.cidr = '192.168.25.0/24'
+        ctlplane_b.id = 'ctlplane_b_id'
+        ctlplane_b.name = 'ctlplane_b'
+        ctlplane_b.cidr = '192.168.26.0/24'
+
+        external_a.id = 'external_a_id'
+        external_a.name = 'external_a'
+        external_a.cidr = '10.0.0.0/24'
+
+        int_api_a.id = 'internal_api_a_id'
+        int_api_a.name = 'internal_api_a'
+        int_api_a.cidr = '172.17.0.0/24'
+        int_api_b.id = 'internal_api_b_id'
+        int_api_b.name = 'internal_api_b'
+        int_api_b.cidr = '172.17.1.0/24'
+
+        self.network.find_network.side_effect = [
+            ctlplane_net, internal_api_net,  # compute-0
+            ctlplane_net, external_net, internal_api_net,  # controller-0
+            ctlplane_net, external_net, internal_api_net,  # controller-1
+            ctlplane_net, external_net, internal_api_net,  # controller-2
+        ]
+        self.network.get_subnet.side_effect = [
+            ctlplane_a, ctlplane_b, int_api_a, int_api_b,  # compute-0
+            ctlplane_a, external_a, int_api_a,  # controller-0,
+            ctlplane_a, external_a, int_api_a,  # controller-1,
+            ctlplane_a, external_a, int_api_a,  # controller-2,
+        ]
+
+        argslist = ['--roles-file', self.roles_file.name,
+                    '--output', self.extract_file.name,
+                    '--yes']
         self.app.command_options = argslist
-        verifylist = [('output', self.extract_file.name), ('yes', True)]
+        verifylist = [('roles_file', self.roles_file.name),
+                      ('output', self.extract_file.name),
+                      ('yes', True)]
 
         parsed_args = self.check_parser(self.cmd,
                                         argslist, verifylist)
@@ -822,6 +927,17 @@ class TestExtractProvisionedNode(test_utils.TestCommand):
             'name': 'Compute',
             'count': 1,
             'hostname_format': '%stackname%-novacompute-%index%',
+            'defaults': {
+                'network_config': {'network_deployment_actions': ['CREATE'],
+                                   'physical_bridge_name': 'br-ex',
+                                   'public_interface_name': 'nic1',
+                                   'template': 'templates/compute.j2'},
+                'networks': [{'network': 'ctlplane',
+                              'subnet': 'ctlplane_b',
+                              'vif': True},
+                             {'network': 'internal_api',
+                              'subnet': 'internal_api_b'}]
+            },
             'instances': [{
                 'hostname': 'overcloud-novacompute-0',
                 'name': 'bm-3'
@@ -830,6 +946,21 @@ class TestExtractProvisionedNode(test_utils.TestCommand):
             'name': 'Controller',
             'count': 3,
             'hostname_format': '%stackname%-controller-%index%',
+            'defaults': {
+                'network_config': {'default_route_network': ['External'],
+                                   'network_deployment_actions': ['CREATE'],
+                                   'networks_skip_config': ['Tenant'],
+                                   'physical_bridge_name': 'br-ex',
+                                   'public_interface_name': 'nic1',
+                                   'template': 'templates/controller.j2'},
+                'networks': [{'network': 'ctlplane',
+                              'subnet': 'ctlplane_a',
+                              'vif': True},
+                             {'network': 'external',
+                              'subnet': 'external_a'},
+                             {'network': 'internal_api',
+                              'subnet': 'internal_api_a'}]
+            },
             'instances': [{
                 'hostname': 'overcloud-controller-0',
                 'name': 'bm-0'
@@ -858,9 +989,9 @@ class TestExtractProvisionedNode(test_utils.TestCommand):
 
         self.baremetal.node.list.return_value = nodes
 
-        argslist = []
+        argslist = ['--roles-file', self.roles_file.name]
         self.app.command_options = argslist
-        verifylist = []
+        verifylist = [('roles_file', self.roles_file.name)]
 
         parsed_args = self.check_parser(self.cmd,
                                         argslist, verifylist)

tripleoclient/v1/overcloud_node.py

@@ -15,12 +15,14 @@
 
 import collections
 import datetime
+import ipaddress
 import json
 import logging
 import os
 import sys
 
 from cliff.formatters import table
+from openstack import exceptions as openstack_exc
 from osc_lib import exceptions as oscexc
 from osc_lib.i18n import _
 from osc_lib import utils
@@ -458,6 +460,7 @@ class ExtractProvisionedNode(command.Command):
         self.clients = self.app.client_manager
         self.orchestration_client = self.clients.orchestration
         self.baremetal_client = self.clients.baremetal
+        self.network_client = self.clients.network
 
     def get_parser(self, prog_name):
         parser = super(ExtractProvisionedNode, self).get_parser(prog_name)
@@ -473,15 +476,51 @@ class ExtractProvisionedNode(command.Command):
         parser.add_argument('-y', '--yes', default=False, action='store_true',
                             help=_('Skip yes/no prompt for existing files '
                                    '(assume yes).'))
+        parser.add_argument('--roles-file', '-r', dest='roles_file',
+                            required=True,
+                            help=_('Role data definition file'))
         return parser
 
+    def _get_subnet_from_net_name_and_ip(self, net_name, ip_addr):
+        try:
+            network = self.network_client.find_network(net_name)
+        except openstack_exc.DuplicateResource:
+            raise oscexc.CommandError(
+                "Unable to extract role networks. Duplicate network resources "
+                "with name %s detected." % net_name)
+
+        if network is None:
+            raise oscexc.CommandError("Unable to extract role networks. "
+                                      "Network %s not found." % net_name)
+
+        for subnet_id in network.subnet_ids:
+            subnet = self.network_client.get_subnet(subnet_id)
+            if (ipaddress.ip_address(ip_addr)
+                    in ipaddress.ip_network(subnet.cidr)):
+                subnet_name = subnet.name
+                return subnet_name
+
+        raise oscexc.CommandError("Unable to extract role networks. Could not "
+                                  "find subnet for IP address %(ip)s on "
+                                  "network %(net)s." % {'ip': ip_addr,
+                                                        'net': net_name})
+
     def take_action(self, parsed_args):
         self.log.debug("take_action(%s)" % parsed_args)
+
+        roles_file = os.path.abspath(parsed_args.roles_file)
+        with open(roles_file, 'r') as fd:
+            role_data = yaml.safe_load(fd.read())
+        # Convert role_data to a dict
+        role_data = {x['name']: x for x in role_data}
+
         self._setup_clients()
         stack = oooutils.get_stack(self.orchestration_client,
                                    parsed_args.stack)
         host_vars = oooutils.get_stack_output_item(
             stack, 'AnsibleHostVarsMap') or {}
+        role_net_ip_map = oooutils.get_stack_output_item(
+            stack, 'RoleNetIpMap') or {}
         parameters = stack.to_dict().get('parameters', {})
 
         # list all baremetal nodes and map hostname to node name
@@ -492,32 +531,102 @@ class ExtractProvisionedNode(command.Command):
             if hostname and node.name:
                 hostname_node_map[hostname] = node.name
 
-        role_data = six.StringIO()
+        data = []
-        role_data.write('# Generated with the following on %s\n#\n' %
+        for role_name, entries in host_vars.items():
-                        datetime.datetime.now().isoformat())
-        role_data.write('#   openstack %s\n#\n\n' %
-                        ' '.join(self.app.command_options))
-        for role, entries in host_vars.items():
             role_count = len(entries)
 
             # skip zero count roles
             if not role_count:
                 continue
 
-            role_data.write('- name: %s\n' % role)
+            if role_name not in role_data:
-            role_data.write('  count: %s\n' % role_count)
+                raise oscexc.CommandError(
+                    "Unable to extract. Invalid role file. Role {} is not "
+                    "defined in roles file {}".format(role_name, roles_file))
 
-            hostname_format = parameters.get('%sHostnameFormat' % role)
+            role = collections.OrderedDict()
+            role['name'] = role_name
+            role['count'] = role_count
+
+            hostname_format = parameters.get('%sHostnameFormat' % role_name)
             if hostname_format:
-                role_data.write('  hostname_format: "%s"\n' % hostname_format)
+                role['hostname_format'] = hostname_format
 
-            role_data.write('  instances:\n')
+            defaults = role['defaults'] = {}
 
+            # Add networks to the role default section
+            role_networks = defaults['networks'] = []
+            for net_name, ips in role_net_ip_map[role_name].items():
+                subnet_name = self._get_subnet_from_net_name_and_ip(net_name,
+                                                                    ips[0])
+                if net_name == constants.CTLPLANE_NET_NAME:
+                    role_networks.append({'network': net_name,
+                                          'subnet': subnet_name,
+                                          'vif': True})
+                else:
+                    role_networks.append({'network': net_name,
+                                          'subnet': subnet_name})
+
+            # Add network config to role defaults section
+            net_conf = defaults['network_config'] = {}
+            net_conf['template'] = parameters.get(
+                role_name + 'NetworkConfigTemplate')
+
+            if parameters.get(role_name + 'NetworkDeploymentActions'):
+                net_conf['network_deployment_actions'] = parameters.get(
+                    role_name + 'NetworkDeploymentActions')
+            else:
+                net_conf['network_deployment_actions'] = parameters.get(
+                    'NetworkDeploymentActions', ['CREATE'])
+
+            if isinstance(net_conf['network_deployment_actions'], str):
+                net_conf['network_deployment_actions'] = net_conf[
+                    'network_deployment_actions'].split(',')
+
+            # The NetConfigDataLookup parameter is of type: json, but when
+            # not set it returns as string '{}'
+            ncdl = parameters.get('NetConfigDataLookup')
+            if isinstance(ncdl, str):
+                ncdl = json.loads(ncdl)
+            if ncdl:
+                net_conf['net_config_data_lookup'] = ncdl
+
+            if parameters.get('DnsSearchDomains'):
+                net_conf['dns_search_domains'] = parameters.get(
+                    'DnsSearchDomains')
+
+            net_conf['physical_bridge_name'] = parameters.get(
+                'NeutronPhysicalBridge', 'br-ex')
+            net_conf['public_interface_name'] = parameters.get(
+                'NeutronPublicInterface', 'nic1')
+
+            if role_data[role_name].get('default_route_networks'):
+                net_conf['default_route_network'] = role_data[role_name].get(
+                    'default_route_networks')
+            if role_data[role_name].get('networks_skip_config'):
+                net_conf['networks_skip_config'] = role_data[role_name].get(
+                    'networks_skip_config')
+
+            # Add individual instances
+            instances = role['instances'] = []
             for entry in sorted(entries):
-                role_data.write('  - hostname: %s\n' % entry)
+                instance = {'hostname': entry}
                 if entry in hostname_node_map:
-                    role_data.write('    name: %s\n' %
+                    instance['name'] = hostname_node_map[entry]
-                                    hostname_node_map[entry])
+                instances.append(instance)
+
+            data.append(role)
+
+        # Write the file header
+        file_data = six.StringIO()
+        file_data.write('# Generated with the following on %s\n#\n' %
+                        datetime.datetime.now().isoformat())
+        file_data.write('#   openstack %s\n#\n\n' %
+                        ' '.join(self.app.command_options))
+        # Write the data
+        if data:
+            yaml.dump(data, file_data, RoleDataDumper, width=120,
+                      default_flow_style=False)
 
         if parsed_args.output:
             if (os.path.exists(parsed_args.output)
@@ -530,5 +639,14 @@ class ExtractProvisionedNode(command.Command):
                         "Will not overwrite existing file:"
                         " %s" % parsed_args.output)
             with open(parsed_args.output, 'w+') as fp:
-                fp.write(role_data.getvalue())
+                fp.write(file_data.getvalue())
-        self.app.stdout.write(role_data.getvalue())
+        self.app.stdout.write(file_data.getvalue())
+
+
+class RoleDataDumper(yaml.SafeDumper):
+    def represent_ordered_dict(self, data):
+        return self.represent_dict(data.items())
+
+
+RoleDataDumper.add_representer(collections.OrderedDict,
+                               RoleDataDumper.represent_ordered_dict)