Browse Source

Merge "Add support for Getting administrative GUID via netlink"

changes/81/707381/1
Zuul 1 week ago
parent
commit
fb9ad9f8a6
9 changed files with 236 additions and 37 deletions
  1. +8
    -0
      networking_mlnx/eswitchd/common/exceptions.py
  2. +2
    -1
      networking_mlnx/eswitchd/eswitch_manager.py
  3. +77
    -36
      networking_mlnx/eswitchd/utils/ib_utils.py
  4. +13
    -0
      networking_mlnx/internal/netdev_ops/api.py
  5. +16
    -0
      networking_mlnx/internal/netdev_ops/exceptions.py
  6. +44
    -0
      networking_mlnx/internal/netdev_ops/impl_pyroute2.py
  7. +10
    -0
      networking_mlnx/internal/netdev_ops/netdev_ops_abs.py
  8. +0
    -0
      networking_mlnx/tests/unit/eswitchd/utils/__init__.py
  9. +66
    -0
      networking_mlnx/tests/unit/eswitchd/utils/test_ib_utils.py

+ 8
- 0
networking_mlnx/eswitchd/common/exceptions.py View File

@@ -28,3 +28,11 @@ class DeviceNotFoundException(MlxException):

def __str__(self):
return 'PCI device not found: %s' % self.message


class InvalidGUIDFormatException(MlxException):
def __init__(self, message=None):
self.message = message

def __str__(self):
return 'Invalid GUID format: %s' % self.message

+ 2
- 1
networking_mlnx/eswitchd/eswitch_manager.py View File

@@ -54,7 +54,8 @@ class PfPciDeviceWrapper(object):

def get_vfs_macs_ib(self, vf_idxs):
return self.ib_utils.get_vfs_macs_ib(
self.mlx_dev_name, self.hca_port, vf_idxs, self.device_type)
self.net_dev_name, self.mlx_dev_name, self.hca_port, vf_idxs,
self.device_type)

def get_vf_state(self, vf_index):
# TODO(adrianc) implement

+ 77
- 36
networking_mlnx/eswitchd/utils/ib_utils.py View File

@@ -18,6 +18,7 @@ import re
from oslo_log import log as logging

from networking_mlnx.eswitchd.common import constants
from networking_mlnx.eswitchd.common import exceptions
from networking_mlnx.eswitchd.utils import pci_utils
from networking_mlnx.internal.netdev_ops import api as net_dev_api
from networking_mlnx.internal.netdev_ops import constants as netdev_const
@@ -34,6 +35,8 @@ class IbUtils(object):
DEFAULT_MASK = 0x7fff
DEFAULT_PKEY = 0xffff
PKEYS_PATH = "/sys/class/infiniband/%s/ports/%s/pkeys/*"
GUID_FMT_MLNX4 = r"^[0-9a-fA-F]{16}$"
GUID_FMT_MLNX5 = r"^([0-9a-fA-F]{2}:){7}([0-9A-Fa-f]{2})$"

def _config_vf_pkey(self, ppkey_idx, pkey_idx,
pf_mlx_dev, vf_pci_id, hca_port):
@@ -66,9 +69,26 @@ class IbUtils(object):
guid = prefix + '00:00:' + suffix
return guid

def get_vfs_macs_ib(self, pf_mlx_name, hca_port, vf_idxs, type):
def _get_mac_from_guid(self, guid):
"""Truncate GUID to mac by removing the 4th and 5th bytes.

:param guid: str formatted in either: xxxxxxxxxxxxxxxx or
xx:xx:xx:xx:xx:xx:xx:xx where 'x' is a hexadecimal digit
:return: mac
"""
if re.match(IbUtils.GUID_FMT_MLNX4, guid):
mac = ":".join(re.findall('..?', guid[:6] + guid[-6:]))
elif re.match(IbUtils.GUID_FMT_MLNX5, guid):
mac = guid[:8] + guid[-9:]
else:
raise exceptions.InvalidGUIDFormatException(guid)
return mac

def get_vfs_macs_ib(self, pf_net_name, pf_mlx_name, hca_port, vf_idxs,
type):
"""Get assigned Infiniband mac address for VFs

:param pf_net_name: PF net device name
:param pf_mlx_name: PF IB device name
:param hca_port: hca port number
:param vf_idxs: list of VF indexes to get mac address
@@ -79,10 +99,12 @@ class IbUtils(object):
macs_map = {}
if type == constants.MLNX4_DEVICE_TYPE:
macs_map.update(
self._get_vfs_macs_ib_mlnx4(pf_mlx_name, hca_port, vf_idxs))
self._get_vfs_macs_ib_mlnx4(pf_net_name, pf_mlx_name,
hca_port, vf_idxs))
elif type == constants.MLNX5_DEVICE_TYPE:
macs_map.update(
self._get_vfs_macs_ib_mlnx5(pf_mlx_name, vf_idxs))
self._get_vfs_macs_ib_mlnx5(pf_net_name, pf_mlx_name,
vf_idxs))
return macs_map

def _get_gid_to_vf_idx_mapping(self, pf_ib_dev, hca_port, vf_idxs):
@@ -102,41 +124,60 @@ class IbUtils(object):
mapping[vf_gid] = vf_idx
return mapping

def _get_vfs_macs_ib_mlnx4(self, pf_mlx_name, hca_port, vf_idxs):
macs_map = {}
gid_idx_to_vf_idx = self._get_gid_to_vf_idx_mapping(
pf_mlx_name, hca_port, vf_idxs)
gid_idxs = gid_idx_to_vf_idx.keys()
guids_path = constants.MLNX4_ADMIN_GUID_PATH % (pf_mlx_name, hca_port,
'[1-9]*')
paths = glob.glob(guids_path)
for path in paths:
gid_index = int(path.split('/')[-1])
if gid_index not in gid_idxs:
continue
with open(path) as f:
guid = f.readline().strip()
if guid == constants.MLNX4_INVALID_GUID:
mac = constants.INVALID_MAC
else:
head = guid[:6]
tail = guid[-6:]
mac = ":".join(re.findall('..?', head + tail))
macs_map[gid_idx_to_vf_idx[gid_index]] = mac
def _get_vfs_ib_mac_netdev_api(self, pf_net_name, vf_idxs):
try:
macs_map = {}
for vf_idx in vf_idxs:
guid = net_dev_api.get_vf_guid(pf_net_name, vf_idx)
macs_map[vf_idx] = self._get_mac_from_guid(guid)
return macs_map
except api_exceptions.NetlinkAttrNotFoundError:
return None

def _get_vfs_macs_ib_mlnx4(self, pf_net_name, pf_mlx_name, hca_port,
vf_idxs):
macs_map = self._get_vfs_ib_mac_netdev_api(pf_net_name, vf_idxs)
# TODO(adrianc): The logic below should be removed once major distros
# have kernel based on 5.5.0 or newer.
if macs_map is None:
LOG.debug("Failed to get vf guid via netdev API, "
"attempting to get vf guid via sysfs.")
macs_map = {}
gid_idx_to_vf_idx = self._get_gid_to_vf_idx_mapping(
pf_mlx_name, hca_port, vf_idxs)
gid_idxs = gid_idx_to_vf_idx.keys()
guids_path = constants.MLNX4_ADMIN_GUID_PATH % (pf_mlx_name,
hca_port,
'[1-9]*')
paths = glob.glob(guids_path)
for path in paths:
gid_index = int(path.split('/')[-1])
if gid_index not in gid_idxs:
continue
with open(path) as f:
guid = f.readline().strip()
if guid == constants.MLNX4_INVALID_GUID:
mac = constants.INVALID_MAC
else:
mac = self._get_mac_from_guid(guid)
macs_map[gid_idx_to_vf_idx[gid_index]] = mac
return macs_map

def _get_vfs_macs_ib_mlnx5(self, pf_mlx_name, vf_idxs):
macs_map = {}
for vf_idx in vf_idxs:
guid_path = (
constants.MLNX5_GUID_NODE_PATH % {'module': pf_mlx_name,
'vf_num': vf_idx})
with open(guid_path) as f:
guid = f.readline().strip()
head = guid[:8]
tail = guid[-9:]
mac = head + tail
macs_map[vf_idx] = mac
def _get_vfs_macs_ib_mlnx5(self, pf_net_name, pf_mlx_name, vf_idxs):
macs_map = self._get_vfs_ib_mac_netdev_api(pf_net_name, vf_idxs)
# TODO(adrianc): The logic below should be removed once major distros
# have kernel based on 5.5.0 or newer.
if macs_map is None:
LOG.debug("Failed to get vf guid via netdev API, "
"attempting to get vf guid via sysfs.")
for vf_idx in vf_idxs:
guid_path = (
constants.MLNX5_GUID_NODE_PATH % {'module': pf_mlx_name,
'vf_num': vf_idx})
with open(guid_path) as f:
guid = f.readline().strip()
mac = self._get_mac_from_guid(guid)
macs_map[vf_idx] = mac
return macs_map

def config_vf_mac_address(self, pf_net_dev, pf_mlx_dev, vf_idx,

+ 13
- 0
networking_mlnx/internal/netdev_ops/api.py View File

@@ -52,3 +52,16 @@ def set_vf_guid(pf_ifname, vf_idx, guid):
where x is a hexadecimal digit.
"""
__netdev_ops.set_vf_guid(pf_ifname, vf_idx, guid)


def get_vf_guid(pf_ifname, vf_idx):
"""Get vf administrative GUID

:param pf_ifname: pf netdev name
:param vf_idx: vf index
:returns vf_guid: 64bit guid str in xx:xx:xx:xx:xx:xx:xx:xx format where
x is a hexadecimal digit.

NOTE: This operation does not need elevated privileges.
"""
return __netdev_ops.get_vf_guid(pf_ifname, vf_idx)

+ 16
- 0
networking_mlnx/internal/netdev_ops/exceptions.py View File

@@ -29,3 +29,19 @@ class NetlinkRuntimeError(RuntimeError):
def __init__(self, e):
message = self.message % {'e': str(e)}
super(NetlinkRuntimeError, self).__init__(message)


class NetlinkAttrNotFoundError(RuntimeError):
message = _("Required netlink attribute was not found. %(e)s")

def __init__(self, e):
message = self.message % {'e': str(e)}
super(NetlinkAttrNotFoundError, self).__init__(message)


class NetlinkUnexpectedAttrValue(RuntimeError):
message = _("Unexpected Netlink attribute value. %(e)s")

def __init__(self, e):
message = self.message % {'e': str(e)}
super(NetlinkUnexpectedAttrValue, self).__init__(message)

+ 44
- 0
networking_mlnx/internal/netdev_ops/impl_pyroute2.py View File

@@ -79,3 +79,47 @@ class IpCommand(netdev_ops_abs.NetDevOperations):
raise exceptions.NetworkInterfaceNotFound(pf_ifname)
except pyroute2.NetlinkError as e:
raise exceptions.NetlinkRuntimeError(e)

def get_vf_guid(self, pf_ifname, vf_idx):
"""Get vf administrative GUID

:param pf_ifname: pf netdev name
:param vf_idx: vf index
:returns vf_guid: 64bit guid str in xx:xx:xx:xx:xx:xx:xx:xx format
where x is a hexadecimal digit.

NOTE: while there are two GUIDs assigned per VF (port and node GUID)
we assume they are the same and return just one value.
"""
try:
ip = pyroute2.IPRoute()
link_idx = ip.link_lookup(ifname=pf_ifname)[0]
attrs = ip.link('get', index=link_idx, ext_mask=1)[0]
except IndexError:
raise exceptions.NetworkInterfaceNotFound(pf_ifname)
except pyroute2.NetlinkError as e:
raise exceptions.NetlinkRuntimeError(e)

vf_attr = (attrs.get_attr('IFLA_VFINFO_LIST').
get_attrs("IFLA_VF_INFO"))[int(vf_idx)]
node_guid_attr = vf_attr.get_attr("IFLA_VF_IB_NODE_GUID")
port_guid_attr = vf_attr.get_attr("IFLA_VF_IB_PORT_GUID")

if node_guid_attr is None or port_guid_attr is None:
# Note(adrianc) both attributes are expected to be present
raise exceptions.NetlinkAttrNotFoundError(
"IFLA_VF_IB_NODE_GUID, IFLA_VF_IB_PORT_GUID")

node_guid = node_guid_attr["ib_node_guid"]
port_guid = port_guid_attr["ib_port_guid"]

if node_guid != port_guid:
# Note(adrianc) both attributes are expected to be the same
raise exceptions.NetlinkUnexpectedAttrValue(
"port and node GUID are expected to be the same for "
"%(netdev)s-vf%(vf_idx)s. actual: %(node_guid)s, %(port_guid)s"
% dict(netdev=pf_ifname,
vf_idx=str(vf_idx),
node_guid=node_guid,
port_guid=port_guid))
return port_guid

+ 10
- 0
networking_mlnx/internal/netdev_ops/netdev_ops_abs.py View File

@@ -49,3 +49,13 @@ class NetDevOperations(object):
:param guid: 64bit guid str in xx:xx:xx:xx:xx:xx:xx:xx format
where x is a hexadecimal digit.
"""

@abc.abstractmethod
def get_vf_guid(self, pf_ifname, vf_idx):
"""Get vf administrative GUID

:param pf_ifname: pf netdev name
:param vf_idx: vf index
:returns guid: 64bit guid str in xx:xx:xx:xx:xx:xx:xx:xx format
where x is a hexadecimal digit.
"""

+ 0
- 0
networking_mlnx/tests/unit/eswitchd/utils/__init__.py View File


+ 66
- 0
networking_mlnx/tests/unit/eswitchd/utils/test_ib_utils.py View File

@@ -0,0 +1,66 @@
# Copyright 2019 Mellanox Technologies, Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import mock

from networking_mlnx.eswitchd.common import exceptions
from networking_mlnx.eswitchd.utils import ib_utils
from networking_mlnx.internal.netdev_ops import exceptions as api_exceptions
from networking_mlnx.tests import base


class TestIbUtils(base.TestCase):

def setUp(self):
super(TestIbUtils, self).setUp()
self.ib_utils = ib_utils.IbUtils()

def test__get_mac_from_guid_mlx4(self):
guid = "1122334455667788"
expected_mac = "11:22:33:66:77:88"
mac = self.ib_utils._get_mac_from_guid(guid)
self.assertEqual(expected_mac, mac)

def test__get_mac_from_guid_mlx5(self):
guid = "11:22:33:44:55:66:77:88"
expected_mac = "11:22:33:66:77:88"
mac = self.ib_utils._get_mac_from_guid(guid)
self.assertEqual(expected_mac, mac)

def test__get_mac_from_guid_invalid_guid(self):
self.assertRaises(exceptions.InvalidGUIDFormatException,
self.ib_utils._get_mac_from_guid, "112233445566")
self.assertRaises(exceptions.InvalidGUIDFormatException,
self.ib_utils._get_mac_from_guid,
"11:22:33:44:55:66")
self.assertRaises(exceptions.InvalidGUIDFormatException,
self.ib_utils._get_mac_from_guid, "rubbish")

@mock.patch.object(ib_utils.net_dev_api, "get_vf_guid")
def test__get_vfs_ib_mac_netlink(self, get_vf_guid_mock):
def mock_call(pf_ifname, vf_idx):
guids = ["11:22:33:44:55:66:77:88", "aa:bb:cc:dd:ee:ff:00:11"]
return guids[vf_idx]
get_vf_guid_mock.side_effect = mock_call

mac_map = self.ib_utils._get_vfs_ib_mac_netdev_api("ib0", [0, 1])
expected_mac_map = {0: "11:22:33:66:77:88",
1: "aa:bb:cc:ff:00:11"}
self.assertDictEqual(expected_mac_map, mac_map)

@mock.patch.object(ib_utils.net_dev_api, "get_vf_guid",
side_effect=api_exceptions.NetlinkAttrNotFoundError(""))
def test__get_vfs_ib_mac_netlink_fail(self, get_vf_guid_mock):
mac_map = self.ib_utils._get_vfs_ib_mac_netdev_api("ib0", [0, 1])
self.assertIsNone(mac_map)

Loading…
Cancel
Save