From afdfa33099e45efa190ae211e78c946154a40687 Mon Sep 17 00:00:00 2001 From: "Markin, Sergiy (sm515x)" Date: Wed, 20 Jul 2022 19:17:19 +0000 Subject: [PATCH] [NIC Card Auto-Detect] NIC Card Auto-Detect This patch adds an ability to match several NIC addresses by one hardware profile by using regex as the address. The regex expression has to have regex: prefix in order to be recognized. Change-Id: I0bb067fb1783725e4ac485683eb898d5fc2d7bf2 --- python/drydock_provisioner/objects/node.py | 49 +- python/tests/unit/test_node_logicalnames.py | 139 ++++ .../deckhand_fullsite_nic_autodetect.yaml | 606 ++++++++++++++++++ 3 files changed, 785 insertions(+), 9 deletions(-) create mode 100644 python/tests/yaml_samples/deckhand_fullsite_nic_autodetect.yaml diff --git a/python/drydock_provisioner/objects/node.py b/python/drydock_provisioner/objects/node.py index 9b891255..1c8383fc 100644 --- a/python/drydock_provisioner/objects/node.py +++ b/python/drydock_provisioner/objects/node.py @@ -18,6 +18,7 @@ from defusedxml.ElementTree import fromstring import logging +import re from oslo_versionedobjects import fields as ovo_fields import drydock_provisioner.error as errors @@ -252,22 +253,52 @@ class BaremetalNode(drydock_provisioner.objects.hostprofile.HostProfile): :param alias_name: String value of the current device alias, it is returned if a logicalname is not found. :param bus_type: String value that is used to find the logicalname. - :param address: String value that is used to find the logicalname. + :param address: String value that is used to find the logicalname. It can be PCI address + to identify a vendor or a regexp to match multiple vendors. The regexp + has to start from regexp: prefix - for example: + regex:0000\\:(19|01)\\:00\\.[0-9]. :return: String value of the logicalname or the alias_name if logicalname is not found. """ - nodes = xml_root.findall(".//node[businfo='" + bus_type + "@" - + address + "'].logicalname") - if len(nodes) >= 1 and nodes[0].text: - if (len(nodes) > 1): - self.logger.info("Multiple nodes found for businfo=%s@%s" % - (bus_type, address)) - for logicalname in reversed(nodes[0].text.split("/")): + if address.find("regex:") != -1: + address_regex = address.replace("regex:", "") + nodes = xml_root.findall(".//node") + counter = 0 + logicalnames = [] + addresses = [] + for node in nodes: + if 'class' in node.attrib: + if node.get('class') == "network": + address = node.find('businfo').text.replace("pci@", "") + addresses.append(address) + if re.match(address_regex, address): + counter += 1 + logicalnames.append(node.find('logicalname').text) + if len(logicalnames) > 1: + self.logger.info( + "Multiple nodes found for businfo=%s@%s" % + (bus_type, address_regex)) + if logicalnames[0]: + logicalname = logicalnames[0] + address = addresses[0] self.logger.debug( "Logicalname build dict: node_name = %s, alias_name = %s, " "bus_type = %s, address = %s, to logicalname = %s" % (self.get_name(), alias_name, bus_type, address, - logicalname)) + logicalname)) return logicalname + else: + nodes = xml_root.findall(".//node[businfo='" + bus_type + "@" + address + "'].logicalname") + if len(nodes) >= 1 and nodes[0].text: + if (len(nodes) > 1): + self.logger.info("Multiple nodes found for businfo=%s@%s" % + (bus_type, address)) + for logicalname in reversed(nodes[0].text.split("/")): + self.logger.debug( + "Logicalname build dict: node_name = %s, alias_name = %s, " + "bus_type = %s, address = %s, to logicalname = %s" % + (self.get_name(), alias_name, bus_type, address, + logicalname)) + return logicalname self.logger.debug( "Logicalname build dict: alias_name = %s, bus_type = %s, address = %s, not found" % (alias_name, bus_type, address)) diff --git a/python/tests/unit/test_node_logicalnames.py b/python/tests/unit/test_node_logicalnames.py index d4932d84..77c412ca 100644 --- a/python/tests/unit/test_node_logicalnames.py +++ b/python/tests/unit/test_node_logicalnames.py @@ -160,3 +160,142 @@ class TestClass(object): # Logicalname is not found, returns the alias assert nodes[0].logicalnames['prim_nic02'] == 'prim_nic02' assert nodes[0].get_logicalname('prim_nic02') == 'prim_nic02' + + def test_apply_logicalnames_nic_autodetect_success(self, input_files, + deckhand_orchestrator, drydock_state, + mock_get_build_data): + """Test node apply_logicalnames to get the proper dictionary""" + input_file = input_files.join("deckhand_fullsite_nic_autodetect.yaml") + + design_ref = "file://%s" % str(input_file) + + xml_example = """ + + + + + + + + Rack Mount Chassis + PowerEdge R720xd (SKU=NotProvided;ModelName=PowerEdge R720xd) + Dell Inc. + 6H5LBY1 + 64 + + + + + + + + SMBIOS version 2.7 + DMI version 2.7 + 32-bit processes + + + + PCI bridge + Xeon E5/Core i7 IIO PCI Express Root Port 2a + Intel Corporation + 2 + pci@0000:00:02.0 + 07 + 32 + 33000000 + + + + + + Message Signalled Interrupts + PCI Express + Power Management + + bus mastering + PCI capabilities listing + + + + + + Ethernet interface + NetXtreme BCM5720 Gigabit Ethernet PCIe + Broadcom Inc. and subsidiaries + 0 + pci@0000:19:00.0 + eno3 + 00 + f0:d4:e2:e5:35:8e + 1000000000 + 64 + 33000000 + + + Ethernet interface + I350 Gigabit Network Connection + Intel Corporation + 0.1 + pci@0000:01:00.0 + eno4 + 01 + e4:43:4b:4d:d3:e7 + 1000000000 + 1000000000 + 32 + 33000000 + + + + + SCSI Disk + PERC H710P + DELL + 2.0.0 + scsi@2:0.0.0 + /dev/sda + 8:0 + 3.13 + 0044016c12771be71900034cfba0a38c + 299439751168 + + + +""" + xml_example = xml_example.replace('\n', '') + + def side_effect(**kwargs): + build_data = objects.BuildData( + node_name="controller01", + task_id="tid", + generator="lshw", + data_format="text/plain", + data_element=xml_example) + return [build_data] + + drydock_state.get_build_data = Mock(side_effect=side_effect) + + design_status, design_data = deckhand_orchestrator.get_effective_site( + design_ref, resolve_aliases=True) + + nodes = design_data.baremetal_nodes + + expected = { + 'primary_boot': 'sda', + 'prim_nic03': 'prim_nic03', + 'prim_nic02': 'eno4', + 'prim_nic01': 'eno3' + } + # Tests the whole dictionary + assert nodes[0].logicalnames == expected + # Makes sure the path and / are both removed from primary_boot + assert nodes[0].logicalnames['primary_boot'] == 'sda' + assert nodes[0].get_logicalname('primary_boot') == 'sda' + # A simple logicalname + assert nodes[0].logicalnames['prim_nic01'] == 'eno3' + assert nodes[0].get_logicalname('prim_nic01') == 'eno3' + assert nodes[0].logicalnames['prim_nic02'] == 'eno4' + assert nodes[0].get_logicalname('prim_nic02') == 'eno4' + # Logicalname is not found, returns the alias + assert nodes[0].logicalnames['prim_nic03'] == 'prim_nic03' + assert nodes[0].get_logicalname('prim_nic03') == 'prim_nic03' diff --git a/python/tests/yaml_samples/deckhand_fullsite_nic_autodetect.yaml b/python/tests/yaml_samples/deckhand_fullsite_nic_autodetect.yaml new file mode 100644 index 00000000..d661ef54 --- /dev/null +++ b/python/tests/yaml_samples/deckhand_fullsite_nic_autodetect.yaml @@ -0,0 +1,606 @@ +#Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#################### +# +# bootstrap_seed.yaml - Site server design definition for physical layer +# +#################### +# version the schema in this file so consumers can rationally parse it +--- +schema: 'drydock/Region/v1' +metadata: + schema: 'metadata/Document/v1' + name: 'sitename' + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + tag_definitions: + - tag: 'test' + definition_type: 'lshw_xpath' + definition: "//node[@id=\"display\"]/'clock units=\"Hz\"' > 1000000000" + repositories: + remove_unlisted: true + docker: + repo_type: apt + url: https://docker.io/repo + distributions: + - ubuntu-xenial + gpgkey: |+ + -----BLAH BLAH----- + STUFF + -----BLAH BLAH----- + authorized_keys: + - | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDENeyO5hLPbLLQRZ0oafTYWs1ieo5Q+XgyZQs51Ju + jDGc8lKlWsg1/6yei2JewKMgcwG2Buu1eqU92Xn1SvMZLyt9GZURuBkyjcfVc/8GiU5QP1Of8B7CV0c + kfUpHWYJ17olTzT61Hgz10ioicBF6cjgQrLNcyn05xoaJHD2Vpf8Unxzi0YzA2e77yRqBo9jJVRaX2q + wUJuZrzb62x3zw8Knz6GGSZBn8xRKLaw1SKFpd1hwvL62GfqX5ZBAT1AYTZP1j8GcAoK8AFVn193SEU + vjSdUFa+RNWuJhkjBRfylJczIjTIFb5ls0jpbA3bMA9DE7lFKVQl6vVwFmiIVBI1 samplekey +--- +schema: 'drydock/NetworkLink/v1' +metadata: + schema: 'metadata/Document/v1' + name: oob + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + bonding: + mode: disabled + mtu: 1500 + linkspeed: 100full + trunking: + mode: disabled + default_network: oob + allowed_networks: + - oob +--- +schema: 'drydock/NetworkLink/v1' +metadata: + schema: 'metadata/Document/v1' + name: pxe + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + bonding: + mode: disabled + mtu: 1500 + linkspeed: auto + trunking: + mode: disabled + default_network: pxe + allowed_networks: + - pxe +--- +schema: 'drydock/NetworkLink/v1' +metadata: + schema: 'metadata/Document/v1' + name: gp + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + bonding: + mode: 802.3ad + hash: layer3+4 + peer_rate: slow + mtu: 9000 + linkspeed: auto + trunking: + mode: 802.1q + default_network: mgmt + allowed_networks: + - public + - private + - mgmt +--- +schema: 'drydock/Rack/v1' +metadata: + schema: 'metadata/Document/v1' + name: rack1 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + tor_switches: + switch01name: + mgmt_ip: 1.1.1.1 + sdn_api_uri: polo+https://api.sdn.example.com/switchmgmt?switch=switch01name + switch02name: + mgmt_ip: 1.1.1.2 + sdn_api_uri: polo+https://api.sdn.example.com/switchmgmt?switch=switch02name + location: + clli: HSTNTXMOCG0 + grid: EG12 + local_networks: + - pxe-rack1 +--- +schema: 'drydock/Network/v1' +metadata: + schema: 'metadata/Document/v1' + name: oob + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + cidr: 172.16.100.0/24 + ranges: + - type: static + start: 172.16.100.15 + end: 172.16.100.254 + dns: + domain: ilo.sitename.att.com + servers: 172.16.100.10 +--- +schema: 'drydock/Network/v1' +metadata: + schema: 'metadata/Document/v1' + name: pxe + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + dhcp_relay: + self_ip: 172.16.0.4 + upstream_target: 172.16.5.5 + mtu: 1500 + cidr: 172.16.0.0/24 + ranges: + - type: dhcp + start: 172.16.0.5 + end: 172.16.0.254 + dns: + domain: admin.sitename.att.com + servers: 172.16.0.10 +--- +schema: 'drydock/Network/v1' +metadata: + schema: 'metadata/Document/v1' + name: mgmt + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + vlan: '100' + mtu: 1500 + cidr: 172.16.1.0/24 + ranges: + - type: static + start: 172.16.1.15 + end: 172.16.1.254 + routes: + - subnet: 0.0.0.0/0 + gateway: 172.16.1.1 + metric: 10 + dns: + domain: mgmt.sitename.example.com + servers: 172.16.1.9,172.16.1.10 +--- +schema: 'drydock/Network/v1' +metadata: + schema: 'metadata/Document/v1' + name: private + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + vlan: '101' + mtu: 9000 + cidr: 172.16.2.0/24 + ranges: + - type: static + start: 172.16.2.15 + end: 172.16.2.254 + dns: + domain: priv.sitename.example.com + servers: 172.16.2.9,172.16.2.10 +--- +schema: 'drydock/Network/v1' +metadata: + schema: 'metadata/Document/v1' + name: public + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + vlan: '102' + mtu: 1500 + cidr: 172.16.3.0/24 + ranges: + - type: static + start: 172.16.3.15 + end: 172.16.3.254 + routes: + - subnet: 0.0.0.0/0 + gateway: 172.16.3.1 + metric: 10 + dns: + domain: sitename.example.com + servers: 8.8.8.8 +--- +schema: 'drydock/HostProfile/v1' +metadata: + schema: 'metadata/Document/v1' + name: defaults + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + oob: + type: ipmi + network: oob + account: admin + credential: admin + storage: + physical_devices: + sda: + labels: + role: rootdisk + partitions: + - name: root + size: 20g + bootable: true + filesystem: + mountpoint: '/' + fstype: 'ext4' + mount_options: 'defaults' + - name: boot + size: 1g + bootable: false + filesystem: + mountpoint: '/boot' + fstype: 'ext4' + mount_options: 'defaults' + sdb: + volume_group: 'log_vg' + volume_groups: + log_vg: + logical_volumes: + - name: 'log_lv' + size: '500m' + filesystem: + mountpoint: '/var/log' + fstype: 'xfs' + mount_options: 'defaults' + platform: + image: 'xenial' + kernel: 'ga-16.04' + kernel_params: + quiet: true + console: ttyS2 + metadata: + owner_data: + foo: bar +--- +schema: 'drydock/HostProfile/v1' +metadata: + schema: 'metadata/Document/v1' + name: 'k8-node' + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + host_profile: defaults + hardware_profile: HPGen9v3 + primary_network: mgmt + interfaces: + pxe: + device_link: pxe + labels: + noconfig: true + slaves: + - prim_nic01 + networks: + - pxe + bond0: + device_link: gp + slaves: + - prim_nic01 + - prim_nic02 + networks: + - mgmt + - private + sriov: + vf_count: 2 + trustedmode: false + metadata: + tags: + - 'test' +--- +schema: 'drydock/BaremetalNode/v1' +metadata: + schema: 'metadata/Document/v1' + name: controller01 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + host_profile: k8-node + interfaces: + bond0: + networks: + - '!private' + addressing: + - network: pxe + address: dhcp + - network: mgmt + address: 172.16.1.20 + - network: public + address: 172.16.3.20 + - network: oob + address: 172.16.100.20 + metadata: + rack: rack1 +--- +schema: 'drydock/BaremetalNode/v1' +metadata: + schema: 'metadata/Document/v1' + name: compute01 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + host_profile: k8-node + addressing: + - network: pxe + address: dhcp + - network: mgmt + address: 172.16.1.21 + - network: private + address: 172.16.2.21 + - network: oob + address: 172.16.100.21 + platform: + kernel_params: + isolcpus: hardwareprofile:cpuset.sriov + hugepagesz: hardwareprofile:hugepages.sriov.size + hugepages: hardwareprofile:hugepages.sriov.count + metadata: + rack: rack2 +--- +schema: 'drydock/BaremetalNode/v1' +metadata: + schema: 'metadata/Document/v1' + name: compute02 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + host_profile: k8-node + addressing: + - network: pxe + address: dhcp + - network: mgmt + address: 172.16.1.23 + - network: private + address: 172.16.2.23 + - network: oob + address: 172.16.100.23 + metadata: + rack: rack3 + owner_data: + foo: baz +--- +schema: 'drydock/HardwareProfile/v1' +metadata: + schema: 'metadata/Document/v1' + name: HPGen9v3 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + vendor: HP + generation: '8' + hw_version: '3' + bios_version: '2.2.3' + boot_mode: bios + bootstrap_protocol: pxe + pxe_interface: 0 + device_aliases: + prim_nic01: + address: 'regex:0000\:(19|01)\:00\.[0-9]' + # address: '0000:19:00.0' + dev_type: 'NetXtreme BCM5720 Gigabit Ethernet Controller or I350 Gigabit Ethernet Controller' + bus_type: 'pci' + prim_nic02: + address: '0000:01:00.0' + dev_type: 'I350 Gigabit Ethernet Controller' + bus_type: 'pci' + prim_nic03: + address: '0000:00:03.0' + dev_type: '82540EM Gigabit Ethernet Controller' + bus_type: 'pci' + primary_boot: + address: '2:0.0.0' + dev_type: 'VBOX HARDDISK' + bus_type: 'scsi' + cpu_sets: + sriov: '2,4' + hugepages: + sriov: + size: '1G' + count: 300 + dpdk: + size: '2M' + count: 530000 +--- +schema: 'drydock/HardwareProfile/v1' +metadata: + schema: 'metadata/Document/v1' + # NOTE(mark-burnett): This duplicate is used as a test to ensure we do not + # include erroneous hardware profiles for nodes when computing logical device + # names. + name: HPGen9v3-duplicate + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + vendor: HP + generation: '8' + hw_version: '3' + bios_version: '2.2.3' + boot_mode: bios + bootstrap_protocol: pxe + pxe_interface: 0 + device_aliases: + prim_nic01: + address: '0000:19:00.0' + dev_type: 'NetXtreme BCM5720 Gigabit Ethernet Controller' + bus_type: 'pci' + prim_nic02: + address: '0000:01:00.1' + dev_type: 'I350 Gigabit Ethernet Controller' + bus_type: 'pci' + primary_boot: + address: '2:0.0.0' + dev_type: 'VBOX HARDDISK' + bus_type: 'scsi' + cpu_sets: + sriov: '2,4' + hugepages: + sriov: + size: '1G' + count: 300 + dpdk: + size: '2M' + count: 530000 +--- +schema: 'drydock/BootAction/v1' +metadata: + schema: 'metadata/Document/v1' + name: hw_filtered + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + signaling: false + node_filter: + filter_set_type: 'union' + filter_set: + - filter_type: 'union' + node_names: + - 'compute01' + assets: + - path: /var/tmp/hello.sh + type: file + permissions: '555' + data: |- + IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 + Jwo= + data_pipeline: + - base64_decode + - utf8_decode + - template + - path: /lib/systemd/system/hello.service + type: unit + permissions: '600' + data: |- + W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 + ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu + dGFyZ2V0Cg== + data_pipeline: + - base64_decode + - utf8_decode +... +--- +schema: 'drydock/BootAction/v1' +metadata: + schema: 'metadata/Document/v1' + name: helloworld + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + assets: + - path: /var/tmp/hello.sh + type: file + permissions: '555' + data: |- + IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 + Jwo= + data_pipeline: + - base64_decode + - utf8_decode + - template + - path: /lib/systemd/system/hello.service + type: unit + permissions: '600' + data: |- + W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 + ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu + dGFyZ2V0Cg== + data_pipeline: + - base64_decode + - utf8_decode + - path: /var/tmp/designref.sh + type: file + permissions: '500' + data: | + {{ action.design_ref }} + {{ action.action_id }} + {{ action.action_key }} + data_pipeline: + - utf8_decode + - template + - type: pkg_list + data: + 2ping: '3.2.1-1' + '0xffff': null +--- +schema: 'drydock/BootAction/v1' +metadata: + schema: 'metadata/Document/v1' + name: hw_filtered2 + storagePolicy: 'cleartext' + labels: + application: 'drydock' +data: + node_filter: + filter_set_type: 'union' + filter_set: + - filter_type: 'union' + node_names: + - 'compute02' + assets: + - path: /var/tmp/hello.sh + type: file + permissions: '555' + data: |- + IyEvYmluL2Jhc2gKCmVjaG8gJ0hlbGxvIFdvcmxkISAtZnJvbSB7eyBub2RlLmhvc3RuYW1lIH19 + Jwo= + data_pipeline: + - base64_decode + - utf8_decode + - template + - path: /lib/systemd/system/hello.service + type: unit + permissions: '600' + data: |- + W1VuaXRdCkRlc2NyaXB0aW9uPUhlbGxvIFdvcmxkCgpbU2VydmljZV0KVHlwZT1vbmVzaG90CkV4 + ZWNTdGFydD0vdmFyL3RtcC9oZWxsby5zaAoKW0luc3RhbGxdCldhbnRlZEJ5PW11bHRpLXVzZXIu + dGFyZ2V0Cg== + data_pipeline: + - base64_decode + - utf8_decode + - path: /var/tmp/designref.sh + type: file + permissions: '500' + data: e3sgYWN0aW9uLmRlc2lnbl9yZWYgfX0K + data_pipeline: + - base64_decode + - utf8_decode + - template +...