diff --git a/puppet-manifests/src/modules/platform/manifests/network.pp b/puppet-manifests/src/modules/platform/manifests/network.pp index 64bd2198b9..6b4373830b 100644 --- a/puppet-manifests/src/modules/platform/manifests/network.pp +++ b/puppet-manifests/src/modules/platform/manifests/network.pp @@ -110,13 +110,32 @@ class platform::addresses ( create_resources('network_address', $address_config, {}) } +define platform::interfaces::sriov_config( + $vf_addrs, + $vf_driver = undef +) { + if $vf_driver != undef { + exec { "load ${vf_driver}": + command => "modprobe ${vf_driver}", + path => '/bin:/sbin:/usr/bin:/usr/sbin', + unless => "egrep -q '^${vf_driver} ' /proc/modules", + logoutput => true + } + -> exec { "sriov-vf-bind-device: ${title}": + command => template('platform/sriov.bind-device.erb'), + logoutput => true + } + } +} class platform::interfaces ( $network_config = {}, $route_config = {}, + $sriov_config = {} ) { create_resources('network_config', $network_config, {}) create_resources('network_route', $route_config, {}) + create_resources('platform::interfaces::sriov_config', $sriov_config, {}) } class platform::network::apply { diff --git a/puppet-manifests/src/modules/platform/templates/sriov.bind-device.erb b/puppet-manifests/src/modules/platform/templates/sriov.bind-device.erb new file mode 100644 index 0000000000..409771d93e --- /dev/null +++ b/puppet-manifests/src/modules/platform/templates/sriov.bind-device.erb @@ -0,0 +1,3 @@ +<%- @vf_addrs.each_with_index do |addr, idx| -%> +/usr/share/openvswitch/scripts/dpdk-devbind.py --bind=<%= @vf_driver -%> <%= addr %> +<%- end -%> \ No newline at end of file diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/iinterface_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/iinterface_shell.py index cd9eee09ef..5ddde98e26 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/iinterface_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/iinterface_shell.py @@ -22,7 +22,7 @@ def _print_iinterface_show(cc, iinterface): 'aemode', 'schedpolicy', 'txhashpolicy', 'uuid', 'ihost_uuid', 'vlan_id', 'uses', 'used_by', - 'created_at', 'updated_at', 'sriov_numvfs'] + 'created_at', 'updated_at', 'sriov_numvfs', 'sriov_vf_driver'] optional_fields = ['ipv4_mode', 'ipv6_mode', 'ipv4_pool', 'ipv6_pool'] rename_fields = [{'field': 'dpdksupport', 'label': 'accelerated'}] network_names = "" @@ -272,13 +272,18 @@ def do_host_if_add(cc, args): dest='sriov_numvfs', metavar='', help='The number of SR-IOV VFs of the interface') +@utils.arg('--vf-driver', + dest='sriov_vf_driver', + metavar='', + choices=['netdevice', 'vfio'], + help='The SR-IOV VF driver for this device') def do_host_if_modify(cc, args): """Modify interface attributes.""" rwfields = ['iftype', 'ifname', 'imtu', 'aemode', 'txhashpolicy', 'datanetworks', 'providernetworks', 'ports', 'ifclass', 'networks', 'ipv4_mode', 'ipv6_mode', 'ipv4_pool', 'ipv6_pool', - 'sriov_numvfs'] + 'sriov_numvfs', 'sriov_vf_driver'] ihost = ihost_utils._find_ihost(cc, args.hostnameorid) diff --git a/sysinv/cgts-client/cgts-client/cgtsclient/v1/port_shell.py b/sysinv/cgts-client/cgts-client/cgtsclient/v1/port_shell.py index 361d2f4180..87591b3ec6 100644 --- a/sysinv/cgts-client/cgts-client/cgtsclient/v1/port_shell.py +++ b/sysinv/cgts-client/cgts-client/cgtsclient/v1/port_shell.py @@ -19,8 +19,8 @@ def _print_port_show(port): fields = ['name', 'namedisplay', 'type', 'pciaddr', 'dev_id', 'numa_node', 'sriov_totalvfs', 'sriov_numvfs', - 'sriov_vfs_pci_address', 'driver', - 'pclass', 'pvendor', 'pdevice', + 'sriov_vfs_pci_address', 'sriov_vf_driver', + 'driver', 'pclass', 'pvendor', 'pdevice', 'capabilities', 'uuid', 'host_uuid', 'interface_uuid', 'dpdksupport', @@ -28,8 +28,8 @@ def _print_port_show(port): labels = ['name', 'namedisplay', 'type', 'pciaddr', 'dev_id', 'processor', 'sriov_totalvfs', 'sriov_numvfs', - 'sriov_vfs_pci_address', 'driver', - 'pclass', 'pvendor', 'pdevice', + 'sriov_vfs_pci_address', 'sriov_vf_driver', + 'driver', 'pclass', 'pvendor', 'pdevice', 'capabilities', 'uuid', 'host_uuid', 'interface_uuid', 'accelerated', diff --git a/sysinv/sysinv/sysinv/etc/sysinv/profileSchema.xsd b/sysinv/sysinv/sysinv/etc/sysinv/profileSchema.xsd index 8e2d3db106..de9488b872 100644 --- a/sysinv/sysinv/sysinv/etc/sysinv/profileSchema.xsd +++ b/sysinv/sysinv/sysinv/etc/sysinv/profileSchema.xsd @@ -162,6 +162,7 @@ + diff --git a/sysinv/sysinv/sysinv/sysinv/agent/manager.py b/sysinv/sysinv/sysinv/sysinv/agent/manager.py index b6d0d6ea41..66f8862be8 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/manager.py @@ -613,6 +613,7 @@ class AgentManager(service.PeriodicService): 'sriov_totalvfs': port.sriov_totalvfs, 'sriov_numvfs': port.sriov_numvfs, 'sriov_vfs_pci_address': port.sriov_vfs_pci_address, + 'sriov_vf_driver': port.sriov_vf_driver, 'driver': port.driver, 'mac': port.mac, 'mtu': port.mtu, diff --git a/sysinv/sysinv/sysinv/sysinv/agent/pci.py b/sysinv/sysinv/sysinv/sysinv/agent/pci.py index 77a175a12b..d1d10761a4 100644 --- a/sysinv/sysinv/sysinv/sysinv/agent/pci.py +++ b/sysinv/sysinv/sysinv/sysinv/agent/pci.py @@ -127,6 +127,7 @@ class Port: self.sriov_totalvfs = kwargs.get('sriov_totalvfs') self.sriov_numvfs = kwargs.get('sriov_numvfs') self.sriov_vfs_pci_address = kwargs.get('sriov_vfs_pci_address') + self.sriov_vf_driver = kwargs.get('sriov_vf_driver') self.driver = kwargs.get('driver') self.dpdksupport = kwargs.get('dpdksupport') @@ -234,6 +235,31 @@ class PCIOperator(object): LOG.debug("sriov_vfs_pci_address: %s" % sriov_vfs_pci_address) return sriov_vfs_pci_address + def get_pci_sriov_vf_driver_name(self, pciaddr, sriov_vfs_pci_address): + vf_driver = None + for addr in sriov_vfs_pci_address: + + try: + with open(os.devnull, "w") as fnull: + output = subprocess.check_output(['lspci', '-vmmks', addr], + stderr=fnull) + except Exception as e: + LOG.error("Error getting PCI data for SR-IOV " + "VF address %s: %s", addr, e) + continue + + for line in output.split('\n'): + pci_attr = shlex.split(line.strip()) + if (pci_attr and len(pci_attr) == 2 and 'Module' in pci_attr[0]): + vf_driver = pci_attr[1] + break + + # All VFs have the same driver per device. + if vf_driver: + break + + return vf_driver + def get_pci_driver_name(self, pciaddr): ddriver = '/sys/bus/pci/devices/' + pciaddr + '/driver/module/drivers' try: @@ -443,6 +469,7 @@ class PCIOperator(object): sriov_totalvfs = self.get_pci_sriov_totalvfs(a) sriov_numvfs = self.get_pci_sriov_numvfs(a) sriov_vfs_pci_address = self.get_pci_sriov_vfs_pci_address(a, sriov_numvfs) + sriov_vf_driver = self.get_pci_sriov_vf_driver_name(a, sriov_vfs_pci_address) driver = self.get_pci_driver_name(a) # Determine DPDK support @@ -597,6 +624,7 @@ class PCIOperator(object): "sriov_numvfs": sriov_numvfs, "sriov_vfs_pci_address": ','.join(str(x) for x in sriov_vfs_pci_address), + "sriov_vf_driver": sriov_vf_driver, "driver": driver, "pci_address": a, "mac": mac, diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/ethernet_port.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/ethernet_port.py index 3d7d296051..cbf294a1ad 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/ethernet_port.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/ethernet_port.py @@ -103,6 +103,9 @@ class EthernetPort(base.APIBase): sriov_vfs_pci_address = wtypes.text "The PCI Addresses of the VFs" + sriov_vf_driver = wtypes.text + "The SR-IOV VF driver for this device" + driver = wtypes.text "The kernel driver for this device" @@ -170,8 +173,8 @@ class EthernetPort(base.APIBase): 'pclass', 'pvendor', 'pdevice', 'psvendor', 'psdevice', 'numa_node', 'mac', 'sriov_totalvfs', 'sriov_numvfs', - 'sriov_vfs_pci_address', 'driver', - 'mtu', 'speed', 'link_mode', + 'sriov_vfs_pci_address', 'sriov_vf_driver', + 'driver', 'mtu', 'speed', 'link_mode', 'duplex', 'autoneg', 'bootp', 'capabilities', 'host_uuid', 'interface_uuid', diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py index 2873a65c27..3273a18cdb 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/host.py @@ -3174,6 +3174,14 @@ class HostController(rest.RestController): host['uuid']) raise wsme.exc.ClientSideError(msg) + for p in ports: + if (interface.sriov_vf_driver == constants.SRIOV_DRIVER_TYPE_NETDEVICE and + p.sriov_vf_driver is None): + msg = (_("Value for SR-IOV VF driver is 'netdevice', but " + "corresponding port has an invalid driver")) + LOG.info(msg) + raise wsme.exc.ClientSideError(msg) + def _semantic_check_unlock_upgrade(self, ihost, force_unlock=False): """ Perform semantic checks related to upgrades prior to unlocking host. diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py index 585beda1cf..c5f3da88fe 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/interface.py @@ -195,6 +195,9 @@ class Interface(base.APIBase): sriov_numvfs = int "The number of configured SR-IOV VFs" + sriov_vf_driver = wtypes.text + "The driver of configured SR-IOV VFs" + networks = [wtypes.text] "Represent the networks of the interface" @@ -223,7 +226,7 @@ class Interface(base.APIBase): 'aemode', 'schedpolicy', 'txhashpolicy', 'vlan_id', 'uses', 'usesmodify', 'used_by', 'ipv4_mode', 'ipv6_mode', 'ipv4_pool', 'ipv6_pool', - 'sriov_numvfs', + 'sriov_numvfs', 'sriov_vf_driver', 'datanetworks']) # never expose the ihost_id attribute @@ -526,12 +529,16 @@ class InterfaceController(rest.RestController): temp_interface['ifclass'] = p['value'] elif '/sriov_numvfs' == p['path']: temp_interface['sriov_numvfs'] = p['value'] + elif '/sriov_vf_driver' == p['path']: + temp_interface['sriov_vf_driver'] = p['value'] + # If network type is not pci-sriov, reset the sriov-numvfs to zero if (temp_interface['sriov_numvfs'] is not None and temp_interface['ifclass'] is not None and temp_interface[ 'ifclass'] != constants.INTERFACE_CLASS_PCI_SRIOV): temp_interface['sriov_numvfs'] = None + temp_interface['sriov_vf_driver'] = None sriov_update = _check_interface_sriov(temp_interface.as_dict(), ihost) @@ -591,6 +598,7 @@ class InterfaceController(rest.RestController): # specific fields are reset as well interface['networktype'] = None interface['sriov_numvfs'] = 0 + interface['sriov_vf_driver'] = None interface['ipv4_mode'] = None interface['ipv6_mode'] = None delete_addressing = True @@ -897,7 +905,8 @@ def _set_defaults(interface): 'aemode': 'active_standby', 'txhashpolicy': None, 'vlan_id': None, - 'sriov_numvfs': 0} + 'sriov_numvfs': 0, + 'sriov_vf_driver': None} networktypelist = [] if interface['ifclass'] == constants.INTERFACE_CLASS_PLATFORM: @@ -1034,6 +1043,14 @@ def _check_interface_sriov(interface, ihost, from_profile=False): "but interface class is not " "pci-sriov.")) + if ('sriov_vf_driver' in interface.keys() and interface['sriov_vf_driver'] + is not None and + ('ifclass' not in interface.keys() or + interface['ifclass'] != constants.INTERFACE_CLASS_PCI_SRIOV)): + raise wsme.exc.ClientSideError(_("SR-IOV VF driver is specified " + "but interface class is not " + "pci-sriov.")) + if ('ifclass' in interface.keys() and interface['ifclass'] == constants.INTERFACE_CLASS_PCI_SRIOV and 'sriov_numvfs' in interface.keys()): @@ -1047,6 +1064,12 @@ def _check_interface_sriov(interface, ihost, from_profile=False): if interface['sriov_numvfs'] <= 0: raise wsme.exc.ClientSideError(_("Value for number of SR-IOV VFs must be > 0.")) + if interface['sriov_vf_driver'] is not None: + if interface['sriov_vf_driver'] not in constants.SRIOV_DRIVER_TYPES: + msg = (_("Value for SR-IOV VF driver must be one of " + "{}").format(', '.join(constants.SRIOV_DRIVER_TYPES))) + raise wsme.exc.ClientSideError(msg) + ports = pecan.request.dbapi.ethernet_port_get_all(hostid=ihost['id']) port_list = [ (p.name, p.sriov_totalvfs, p.driver) for p in ports @@ -1062,10 +1085,10 @@ def _check_interface_sriov(interface, ihost, from_profile=False): if int(interface['sriov_numvfs']) > sriov_totalvfs: raise wsme.exc.ClientSideError(_("The interface support a maximum of %s VFs" % sriov_totalvfs)) - driver = port_list[0][2] if driver is None or not driver: raise wsme.exc.ClientSideError(_("Corresponding port has invalid driver")) + sriov_update = True return sriov_update diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/port.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/port.py index b94e0fe696..449e998c76 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/port.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/port.py @@ -101,6 +101,9 @@ class Port(base.APIBase): sriov_vfs_pci_address = wtypes.text "The PCI Addresses of the VFs" + sriov_vf_driver = wtypes.text + "The SR-IOV VF driver for this device" + driver = wtypes.text "The kernel driver for this device" @@ -150,8 +153,8 @@ class Port(base.APIBase): 'pclass', 'pvendor', 'pdevice', 'psvendor', 'psdevice', 'numa_node', 'sriov_totalvfs', 'sriov_numvfs', - 'sriov_vfs_pci_address', 'driver', - 'capabilities', + 'sriov_vfs_pci_address', 'sriov_vf_driver', + 'driver', 'capabilities', 'host_uuid', 'interface_uuid', 'node_uuid', 'dpdksupport', 'created_at', 'updated_at']) diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile.py index a060f59f4f..549750c501 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile.py @@ -78,7 +78,7 @@ INTERFACE_PROFILE_FIELDS = ['ifname', 'iftype', 'imtu', 'networktype', 'txhashpolicy', 'forihostid', 'datanetworks', 'vlan_id', 'ipv4_mode', 'ipv6_mode', 'ipv4_pool', 'ipv6_pool', - 'sriov_numvfs'] + 'sriov_numvfs', 'sriov_vf_driver'] class Profile(base.APIBase): @@ -1353,6 +1353,7 @@ def _create_if_profile(profile_name, profile_node): 'ipv4_pool': ipv4_mode['pool'], 'ipv6_pool': ipv6_mode['pool'], 'sriov_numvfs': ethIf.virtualFunctions, + 'sriov_vf_driver': ethIf.virtualFunctionDriver, 'interface_profile': True } newIf = interface_api._create(idict, from_profile=True) @@ -1394,6 +1395,7 @@ def _create_if_profile(profile_name, profile_node): 'ipv6_pool': ipv6_pool, 'imtu': aeIf.mtu, 'sriov_numvfs': ethIf.virtualFunctions, + 'sriov_vf_driver': ethIf.virtualFunctionDriver, 'interface_profile': True } @@ -1421,6 +1423,7 @@ def _create_if_profile(profile_name, profile_node): 'ipv6_pool': ipv6_pool, 'imtu': vlanIf.mtu, 'sriov_numvfs': ethIf.virtualFunctions, + 'sriov_vf_driver': ethIf.virtualFunctionDriver, 'interface_profile': True } diff --git a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile_utils.py b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile_utils.py index 993cbc85af..6fa352520a 100644 --- a/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile_utils.py +++ b/sysinv/sysinv/sysinv/sysinv/api/controllers/v1/profile_utils.py @@ -135,6 +135,7 @@ class PciSriov(Network): def __init__(self, node): super(PciSriov, self).__init__(node, constants.NETWORK_TYPE_PCI_SRIOV) self.virtualFunctions = int(node.get('virtualFunctions')) + self.virtualFunctionDriver = node.get('virtualFunctionDriver') class Interface(object): @@ -148,6 +149,7 @@ class Interface(object): self.ipv6Mode = {'mode': None, 'pool': None} self.routes = [] self.virtualFunctions = 0 + self.virtualFunctionDriver = None networksNode = ifNode.find('networks') if networksNode is not None: for netNode in networksNode: @@ -168,6 +170,7 @@ class Interface(object): self.routes = network.routes elif network.networkType == constants.NETWORK_TYPE_PCI_SRIOV: self.virtualFunctions = network.virtualFunctions + self.virtualFunctionDriver = network.virtualFunctionDriver if isinstance(network, Network): self.providerNetworks = network.providerNetworks diff --git a/sysinv/sysinv/sysinv/sysinv/common/constants.py b/sysinv/sysinv/sysinv/sysinv/common/constants.py index c42408fe47..f71d7356fe 100644 --- a/sysinv/sysinv/sysinv/sysinv/common/constants.py +++ b/sysinv/sysinv/sysinv/sysinv/common/constants.py @@ -633,6 +633,11 @@ PLATFORM_NETWORK_TYPES = [NETWORK_TYPE_PXEBOOT, PCI_NETWORK_TYPES = [NETWORK_TYPE_PCI_PASSTHROUGH, NETWORK_TYPE_PCI_SRIOV] +SRIOV_DRIVER_TYPE_VFIO = 'vfio' +SRIOV_DRIVER_TYPE_NETDEVICE = 'netdevice' +SRIOV_DRIVER_TYPES = [SRIOV_DRIVER_TYPE_VFIO, + SRIOV_DRIVER_TYPE_NETDEVICE] + INTERFACE_TYPE_ETHERNET = 'ethernet' INTERFACE_TYPE_VLAN = 'vlan' INTERFACE_TYPE_AE = 'ae' diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index 7e38a20f47..e74c4e0a29 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -2031,6 +2031,8 @@ class ConductorManager(service.PeriodicService): 'sriov_numvfs': inic['sriov_numvfs'], 'sriov_vfs_pci_address': inic['sriov_vfs_pci_address'], + 'sriov_vf_driver': + inic['sriov_vf_driver'], 'driver': inic['driver'], 'dpdksupport': inic['dpdksupport'], 'speed': inic['speed'], diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/085_sriov_vf_driver.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/085_sriov_vf_driver.py new file mode 100644 index 0000000000..879728a9cb --- /dev/null +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/migrate_repo/versions/085_sriov_vf_driver.py @@ -0,0 +1,27 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2019 Wind River Systems, Inc. +# +# SPDX-License-Identifier: Apache-2.0 +# + +from sqlalchemy import Column, MetaData, String, Table + +ENGINE = 'InnoDB' +CHARSET = 'utf8' + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + interfaces = Table('interfaces', meta, autoload=True) + interfaces.create_column(Column('sriov_vf_driver', String(255))) + + ports = Table('ports', meta, autoload=True) + ports.create_column(Column('sriov_vf_driver', String(255))) + + +def downgrade(migrate_engine): + # Downgrade is unsupported in this release. + raise NotImplementedError('SysInv database downgrade is unsupported.') diff --git a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py index 0ff7c2b0be..029c64a965 100644 --- a/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py +++ b/sysinv/sysinv/sysinv/sysinv/db/sqlalchemy/models.py @@ -343,6 +343,7 @@ class Interfaces(Base): ifcapabilities = Column(JSONEncodedDict) farend = Column(JSONEncodedDict) sriov_numvfs = Column(Integer) + sriov_vf_driver = Column(String(255)) used_by = relationship( "Interfaces", @@ -450,6 +451,7 @@ class Ports(Base): dev_id = Column(Integer) sriov_totalvfs = Column(Integer) sriov_numvfs = Column(Integer) + sriov_vf_driver = Column(String(255)) # Each PCI Address is 12 char, 1020 char is enough for 64 devices sriov_vfs_pci_address = Column(String(1020)) driver = Column(String(255)) diff --git a/sysinv/sysinv/sysinv/sysinv/objects/interface.py b/sysinv/sysinv/sysinv/sysinv/objects/interface.py index ffa918109f..9f510f2f73 100644 --- a/sysinv/sysinv/sysinv/sysinv/objects/interface.py +++ b/sysinv/sysinv/sysinv/sysinv/objects/interface.py @@ -145,7 +145,8 @@ class Interface(base.SysinvObject): 'ipv6_mode': utils.ipv6_mode_or_none, 'ipv4_pool': utils.uuid_or_none, 'ipv6_pool': utils.uuid_or_none, - 'sriov_numvfs': utils.int_or_none + 'sriov_numvfs': utils.int_or_none, + 'sriov_vf_driver': utils.str_or_none } _foreign_fields = {'uses': _get_interface_name_list, diff --git a/sysinv/sysinv/sysinv/sysinv/objects/port.py b/sysinv/sysinv/sysinv/sysinv/objects/port.py index 3b605113a6..de09e360b5 100644 --- a/sysinv/sysinv/sysinv/sysinv/objects/port.py +++ b/sysinv/sysinv/sysinv/sysinv/objects/port.py @@ -41,6 +41,7 @@ class Port(base.SysinvObject): 'sriov_totalvfs': utils.int_or_none, 'sriov_numvfs': utils.int_or_none, 'sriov_vfs_pci_address': utils.str_or_none, + 'sriov_vf_driver': utils.str_or_none, 'driver': utils.str_or_none, 'capabilities': utils.dict_or_none, } diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py index 529b039162..fca96a18d0 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/interface.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/interface.py @@ -19,6 +19,7 @@ from sysinv.conductor import openstack from sysinv.openstack.common import log from sysinv.puppet import base +from sysinv.puppet import quoted_str LOG = log.getLogger(__name__) @@ -56,6 +57,7 @@ DHCP_METHOD = 'dhcp' NETWORK_CONFIG_RESOURCE = 'platform::interfaces::network_config' ROUTE_CONFIG_RESOURCE = 'platform::interfaces::route_config' +SRIOV_CONFIG_RESOURCE = 'platform::interfaces::sriov_config' ADDRESS_CONFIG_RESOURCE = 'platform::addresses::address_config' @@ -88,6 +90,7 @@ class InterfacePuppet(base.BasePuppet): NETWORK_CONFIG_RESOURCE: {}, ROUTE_CONFIG_RESOURCE: {}, ADDRESS_CONFIG_RESOURCE: {}, + SRIOV_CONFIG_RESOURCE: {}, } system = self._get_system() @@ -892,6 +895,38 @@ def get_route_config(route, ifname): return config +def get_sriov_config(context, iface): + vf_driver = iface['sriov_vf_driver'] + port = get_interface_port(context, iface) + vf_addr_list = port['sriov_vfs_pci_address'] + + if not vf_addr_list: + return {} + + if vf_driver: + if "vfio" in vf_driver: + vf_driver = "vfio-pci" + elif "netdevice" in vf_driver: + if port['sriov_vf_driver'] is not None: + vf_driver = port['sriov_vf_driver'] + else: + # Should not happen, but in this case the vf driver + # will be determined by the kernel. That is, + # no explicit bind will be performed by Puppet. + vf_driver = None + + # Format the vf addresses as quoted strings in order to prevent + # puppet from treating the address as a time/date value + vf_addrs = [quoted_str(addr.strip()) for addr in vf_addr_list.split(",")] + + config = { + 'ifname': iface['ifname'], + 'vf_driver': vf_driver, + 'vf_addrs': vf_addrs + } + return config + + def get_common_network_config(context, iface, config, network_id=None): """ Augments a basic config dictionary with the attributes specific to an upper @@ -989,6 +1024,14 @@ def generate_network_config(context, config, iface): route_config['name']: route_config }) + interface_class = iface['ifclass'] + if interface_class == constants.INTERFACE_CLASS_PCI_SRIOV: + sriov_config = get_sriov_config(context, iface) + if sriov_config: + config[SRIOV_CONFIG_RESOURCE].update({ + sriov_config['ifname']: format_sriov_config(sriov_config) + }) + def find_network_by_pool_uuid(context, pool_uuid): for networktype, network in six.iteritems(context['networks']): @@ -1188,3 +1231,13 @@ def format_network_config(config): network_config = copy.copy(config) del network_config['ifname'] return network_config + + +def format_sriov_config(config): + """ + Converts a sriov_config resource dictionary to the equivalent puppet + resource definition parameters. + """ + sriov_config = copy.copy(config) + del sriov_config['ifname'] + return sriov_config diff --git a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py index ba09b042bc..207772972c 100644 --- a/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py +++ b/sysinv/sysinv/sysinv/sysinv/puppet/kubernetes.py @@ -250,11 +250,15 @@ class KubernetesPuppet(base.BasePuppet): # Add to the list of pci addreses for this data network resource['rootDevices'].append(port['pciaddr']) else: + device_type = iface.get('sriov_vf_driver', None) + if not device_type: + device_type = constants.SRIOV_DRIVER_TYPE_NETDEVICE + # PCI addresses don't exist for this data network yet resource = {dn_name: { "resourceName": "{}_net_{}".format( ifclass, dn_name).replace("-", "_"), - "deviceType": "netdevice", + "deviceType": device_type, "rootDevices": [port['pciaddr']], "sriovMode": ifclass == constants.INTERFACE_CLASS_PCI_SRIOV diff --git a/sysinv/sysinv/sysinv/sysinv/tests/api/test_interface.py b/sysinv/sysinv/sysinv/sysinv/tests/api/test_interface.py index a40a9b5966..0deeb425c1 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/api/test_interface.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/api/test_interface.py @@ -1121,6 +1121,38 @@ class TestPatch(InterfaceTestCase): self.assertEqual('application/json', response.content_type) self.assertTrue(response.json['error_message']) + def _create_sriov_vf_driver_valid(self, vf_driver, expect_errors=False): + interface = dbutils.create_test_interface(forihostid='1', + datanetworks='group0-data0') + dbutils.create_test_ethernet_port( + id=1, name='eth1', host_id=1, interface_id=interface.id, + pciaddr='0000:00:00.11', dev_id=0, sriov_totalvfs=1, sriov_numvfs=1, + driver='i40e', + sriov_vf_driver='i40evf') + response = self.patch_dict_json( + '%s' % self._get_path(interface['uuid']), + networktype=constants.NETWORK_TYPE_PCI_SRIOV, + ifclass=constants.INTERFACE_CLASS_PCI_SRIOV, + sriov_numvfs=1, + sriov_vf_driver=vf_driver, + expect_errors=expect_errors) + self.assertEqual('application/json', response.content_type) + if expect_errors: + self.assertEqual(http_client.BAD_REQUEST, response.status_int) + self.assertTrue(response.json['error_message']) + else: + self.assertEqual(http_client.OK, response.status_code) + self.assertEqual(vf_driver, response.json['sriov_vf_driver']) + + def test_create_sriov_vf_driver_netdevice_valid(self): + self._create_sriov_vf_driver_valid('netdevice') + + def test_create_sriov_vf_driver_vfio_valid(self): + self._create_sriov_vf_driver_valid('vfio') + + def test_create_sriov_vf_driver_invalid(self): + self._create_sriov_vf_driver_valid('bad_driver', expect_errors=True) + # No longer requires setting the network type back to none # Expected error: The network type of an interface cannot be changed without # first being reset back to none diff --git a/sysinv/sysinv/sysinv/sysinv/tests/db/sqlalchemy/test_migrations.py b/sysinv/sysinv/sysinv/sysinv/tests/db/sqlalchemy/test_migrations.py index 355fbe7fef..57afd86653 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/db/sqlalchemy/test_migrations.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/db/sqlalchemy/test_migrations.py @@ -1894,3 +1894,23 @@ class TestMigrations(BaseMigrationTestCase, WalkVersionsMixin): for col, coltype in ptp_cols.items(): self.assertTrue(isinstance(ptp.c[col].type, getattr(sqlalchemy.types, coltype))) + + def _check_085(self, engine, data): + # 085_sriov_vf_driver.py + + # Assert data types for new columns in tables "interfaces" and "ports" + interfaces = db_utils.get_table(engine, 'interfaces') + interfaces_col = { + 'sriov_vf_driver': 'String', + } + for col, coltype in interfaces_col.items(): + self.assertTrue(isinstance(interfaces.c[col].type, + getattr(sqlalchemy.types, coltype))) + + ports = db_utils.get_table(engine, 'ports') + ports_col = { + 'sriov_vf_driver': 'String', + } + for col, coltype in ports_col.items(): + self.assertTrue(isinstance(ports.c[col].type, + getattr(sqlalchemy.types, coltype))) diff --git a/sysinv/sysinv/sysinv/sysinv/tests/db/utils.py b/sysinv/sysinv/sysinv/sysinv/tests/db/utils.py index f3831e58ca..50bcf835f2 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/db/utils.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/db/utils.py @@ -563,6 +563,7 @@ def get_test_port(**kw): 'sriov_totalvfs': kw.get('sriov_totalvfs'), 'sriov_numvfs': kw.get('sriov_numvfs'), 'sriov_vfs_pci_address': kw.get('sriov_vfs_pci_address'), + 'sriov_vf_driver': kw.get('sriov_vf_driver'), 'driver': kw.get('driver'), 'capabilities': kw.get('capabilities'), 'created_at': kw.get('created_at'), @@ -604,6 +605,7 @@ def get_test_ethernet_port(**kw): 'dev_id': kw.get('dev_id'), 'sriov_totalvfs': kw.get('sriov_totalvfs'), 'sriov_numvfs': kw.get('sriov_numvfs'), + 'sriov_vf_driver': kw.get('sriov_vf_driver'), 'driver': kw.get('driver') } return ethernet_port @@ -685,6 +687,7 @@ def post_get_test_interface(**kw): 'ipv4_pool': kw.get('ipv4_pool'), 'ipv6_pool': kw.get('ipv6_pool'), 'sriov_numvfs': kw.get('sriov_numvfs', None), + 'sriov_vf_driver': kw.get('sriov_vf_driver', None), } return interface @@ -720,6 +723,7 @@ def get_test_interface(**kw): 'ipv4_pool': kw.get('ipv4_pool'), 'ipv6_pool': kw.get('ipv6_pool'), 'sriov_numvfs': kw.get('sriov_numvfs', None), + 'sriov_vf_driver': kw.get('sriov_vf_driver', None) } return interface diff --git a/sysinv/sysinv/sysinv/sysinv/tests/puppet/test_interface.py b/sysinv/sysinv/sysinv/sysinv/tests/puppet/test_interface.py index 513ae78957..df15793ec5 100644 --- a/sysinv/sysinv/sysinv/sysinv/tests/puppet/test_interface.py +++ b/sysinv/sysinv/sysinv/sysinv/tests/puppet/test_interface.py @@ -14,6 +14,7 @@ from sysinv.common import constants from sysinv.common import utils from sysinv.puppet import interface from sysinv.puppet import puppet +from sysinv.puppet import quoted_str from sysinv.objects import base as objbase from sysinv.tests.db import base as dbbase @@ -160,7 +161,8 @@ class BaseTestCase(dbbase.DbTestCase): 'networks': networks, 'networktype': networktype, 'imtu': 1500, - 'sriov_numvfs': kwargs.get('sriov_numvfs', 0)} + 'sriov_numvfs': kwargs.get('sriov_numvfs', 0), + 'sriov_vf_driver': kwargs.get('sriov_vf_driver', None)} db_interface = dbutils.create_test_interface(**interface) self.interfaces.append(db_interface) @@ -175,7 +177,9 @@ class BaseTestCase(dbbase.DbTestCase): 'dpdksupport': kwargs.get('dpdksupport', True), 'pciaddr': kwargs.get('pciaddr', '0000:00:00.' + str(port_id + 1)), - 'dev_id': kwargs.get('dev_id', 0)} + 'dev_id': kwargs.get('dev_id', 0), + 'sriov_vf_driver': kwargs.get('sriov_vf_driver', None), + 'sriov_vfs_pci_address': kwargs.get('sriov_vfs_pci_address', '')} db_port = dbutils.create_test_ethernet_port(**port) self.ports.append(db_port) self._setup_address_and_routes(db_interface) @@ -1103,6 +1107,13 @@ class InterfaceTestCase(BaseTestCase): 'options': 'metric ' + str(metric)} return config + def _get_sriov_config(self, ifname='default', vf_driver='vfio', + vf_addrs=[""]): + config = {'ifname': ifname, + 'vf_driver': vf_driver, + 'vf_addrs': vf_addrs} + return config + def _get_loopback_config(self): network_config = self._get_network_config( ifname=interface.LOOPBACK_IFNAME, method=interface.LOOPBACK_METHOD) @@ -1355,6 +1366,55 @@ class InterfaceTestCase(BaseTestCase): print(expected) self.assertEqual(expected, config) + def _create_sriov_vf_driver_config(self, iface_vf_driver, port_vf_driver, vf_addr_list): + self.iface['ifclass'] = constants.INTERFACE_CLASS_PCI_SRIOV + self.iface['networktype'] = constants.NETWORK_TYPE_PCI_SRIOV + self.iface['sriov_vf_driver'] = iface_vf_driver + self.port['sriov_vf_driver'] = port_vf_driver + self.port['sriov_vfs_pci_address'] = vf_addr_list + self._update_context() + config = interface.get_sriov_config(self.context, self.iface) + return config + + def test_get_sriov_config_netdevice(self): + vf_addr1 = "0000:81:00.0" + vf_addr2 = "0000:81:01.0" + vf_addr_list = "{},{}".format(vf_addr1, vf_addr2) + + config = self._create_sriov_vf_driver_config( + constants.SRIOV_DRIVER_TYPE_NETDEVICE, 'i40evf', vf_addr_list) + expected = self._get_sriov_config( + self.iface['ifname'], 'i40evf', + [quoted_str(vf_addr1), + quoted_str(vf_addr2)]) + self.assertEqual(expected, config) + + def test_get_sriov_config_vfio(self): + vf_addr1 = "0000:81:00.0" + vf_addr2 = "0000:81:01.0" + vf_addr_list = "{},{}".format(vf_addr1, vf_addr2) + + config = self._create_sriov_vf_driver_config( + constants.SRIOV_DRIVER_TYPE_VFIO, 'i40evf', vf_addr_list) + expected = self._get_sriov_config( + self.iface['ifname'], 'vfio-pci', + [quoted_str(vf_addr1), + quoted_str(vf_addr2)]) + self.assertEqual(expected, config) + + def test_get_sriov_config_default(self): + vf_addr1 = "0000:81:00.0" + vf_addr2 = "0000:81:01.0" + vf_addr_list = "{},{}".format(vf_addr1, vf_addr2) + + config = self._create_sriov_vf_driver_config( + None, 'i40evf', vf_addr_list) + expected = self._get_sriov_config( + self.iface['ifname'], None, + [quoted_str(vf_addr1), + quoted_str(vf_addr2)]) + self.assertEqual(expected, config) + def test_is_a_mellanox_cx3_device_false(self): self.assertFalse( interface.is_a_mellanox_cx3_device(self.context, self.iface))