pci unify management

bp pcie-unified-management
Change-Id: I2a285b61c4f41e999670715dfa2cef8c910467b9

Change-Id: Iaddcd2d5dfef987b3d11d30a76e2775c8e60ba38

cyborg/accelerator/drivers/pci/base.py

@@ -0,0 +1,29 @@
+# Copyright 2024 Inspur.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""
+Cyborg Pci driver implementation.
+"""
+
+
+class PciDriver(object):
+    """Base class for Pci drivers.
+    """
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def discover(self):
+        raise NotImplementedError()

cyborg/accelerator/drivers/pci/devspec.py

@@ -0,0 +1,290 @@
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+import re
+import string
+
+import six
+
+from cyborg.accelerator.drivers.pci import utils
+from cyborg.common import exception
+from cyborg.common.i18n import _
+
+MAX_VENDOR_ID = 0xFFFF
+MAX_PRODUCT_ID = 0xFFFF
+MAX_FUNC = 0x7
+MAX_DOMAIN = 0xFFFF
+MAX_BUS = 0xFF
+MAX_SLOT = 0x1F
+ANY = '*'
+REGEX_ANY = '.*'
+
+
+@six.add_metaclass(abc.ABCMeta)
+class PciAddressSpec(object):
+    """Abstract class for all PCI address spec styles
+
+    This class checks the address fields of the pci.passthrough_whitelist
+    """
+
+    @abc.abstractmethod
+    def match(self, pci_addr):
+        pass
+
+    def is_single_address(self):
+        return all([
+            all(c in string.hexdigits for c in self.domain),
+            all(c in string.hexdigits for c in self.bus),
+            all(c in string.hexdigits for c in self.slot),
+            all(c in string.hexdigits for c in self.func)])
+
+    def _set_pci_dev_info(self, prop, maxval, hex_value):
+        a = getattr(self, prop)
+        if a == ANY:
+            return
+        try:
+            v = int(a, 16)
+        except ValueError:
+            raise exception.PciConfigInvalidWhitelist(
+                reason=_("property %(property)s ('%(attr)s') does not parse "
+                         "as a hex number.") % {'property': prop, 'attr': a})
+        if v > maxval:
+            raise exception.PciConfigInvalidWhitelist(
+                reason=_("property %(property)s (%(attr)s) is greater than "
+                         "the maximum allowable value (%(max)X).") % {
+                    'property': prop, 'attr': a, 'max': maxval})
+        setattr(self, prop, hex_value % v)
+
+
+class PhysicalPciAddress(PciAddressSpec):
+    """Manages the address fields for a fully-qualified PCI address.
+
+    This function class will validate the address fields for a single
+    PCI device.
+    """
+    def __init__(self, pci_addr):
+        try:
+            if isinstance(pci_addr, dict):
+                self.domain = pci_addr['domain']
+                self.bus = pci_addr['bus']
+                self.slot = pci_addr['slot']
+                self.func = pci_addr['function']
+            else:
+                self.domain, self.bus, self.slot, self.func = (
+                    utils.get_pci_address_fields(pci_addr))
+            self._set_pci_dev_info('func', MAX_FUNC, '%1x')
+            self._set_pci_dev_info('domain', MAX_DOMAIN, '%04x')
+            self._set_pci_dev_info('bus', MAX_BUS, '%02x')
+            self._set_pci_dev_info('slot', MAX_SLOT, '%02x')
+        except (KeyError, ValueError):
+            raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
+
+    def match(self, phys_pci_addr):
+        conditions = [
+            self.domain == phys_pci_addr.domain,
+            self.bus == phys_pci_addr.bus,
+            self.slot == phys_pci_addr.slot,
+            self.func == phys_pci_addr.func,
+            ]
+        return all(conditions)
+
+
+class PciAddressGlobSpec(PciAddressSpec):
+    """Manages the address fields with glob style.
+
+    This function class will validate the address fields with glob style,
+    check for wildcards, and insert wildcards where the field is left blank.
+    """
+
+    def __init__(self, pci_addr):
+        self.domain = ANY
+        self.bus = ANY
+        self.slot = ANY
+        self.func = ANY
+
+        dbs, sep, func = pci_addr.partition('.')
+        if func:
+            self.func = func.strip()
+            self._set_pci_dev_info('func', MAX_FUNC, '%01x')
+        if dbs:
+            dbs_fields = dbs.split(':')
+            if len(dbs_fields) > 3:
+                raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
+            # If we got a partial address like ":00.", we need to turn this
+            # into a domain of ANY, a bus of ANY, and a slot of 00. This code
+            # allows the address bus and/or domain to be left off
+            dbs_all = [ANY] * (3 - len(dbs_fields))
+            dbs_all.extend(dbs_fields)
+            dbs_checked = [s.strip() or ANY for s in dbs_all]
+            self.domain, self.bus, self.slot = dbs_checked
+            self._set_pci_dev_info('domain', MAX_DOMAIN, '%04x')
+            self._set_pci_dev_info('bus', MAX_BUS, '%02x')
+            self._set_pci_dev_info('slot', MAX_SLOT, '%02x')
+
+    def match(self, phys_pci_addr):
+        conditions = [
+            self.domain in (ANY, phys_pci_addr.domain),
+            self.bus in (ANY, phys_pci_addr.bus),
+            self.slot in (ANY, phys_pci_addr.slot),
+            self.func in (ANY, phys_pci_addr.func)
+            ]
+        return all(conditions)
+
+
+class PciAddressRegexSpec(PciAddressSpec):
+    """Manages the address fields with regex style.
+
+    This function class will validate the address fields with regex style.
+    The validation includes check for all PCI address attributes and validate
+    their regex.
+    """
+    def __init__(self, pci_addr):
+        try:
+            self.domain = pci_addr.get('domain', REGEX_ANY)
+            self.bus = pci_addr.get('bus', REGEX_ANY)
+            self.slot = pci_addr.get('slot', REGEX_ANY)
+            self.func = pci_addr.get('function', REGEX_ANY)
+            self.domain_regex = re.compile(self.domain)
+            self.bus_regex = re.compile(self.bus)
+            self.slot_regex = re.compile(self.slot)
+            self.func_regex = re.compile(self.func)
+        except re.error:
+            raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
+
+    def match(self, phys_pci_addr):
+        conditions = [
+            bool(self.domain_regex.match(phys_pci_addr.domain)),
+            bool(self.bus_regex.match(phys_pci_addr.bus)),
+            bool(self.slot_regex.match(phys_pci_addr.slot)),
+            bool(self.func_regex.match(phys_pci_addr.func))
+            ]
+        return all(conditions)
+
+
+class WhitelistPciAddress(object):
+    """Manages the address fields of the whitelist.
+
+    This class checks the address fields of the pci.passthrough_whitelist
+    configuration option, validating the address fields.
+    Example configs:
+
+        | [pci]
+        | passthrough_whitelist = {"address":"*:0a:00.*",
+        |                          "physical_network":"physnet1"}
+        | passthrough_whitelist = {"address": {"domain": ".*",
+                                               "bus": "02",
+                                               "slot": "01",
+                                               "function": "[0-2]"},
+                                    "physical_network":"net1"}
+        | passthrough_whitelist = {"vendor_id":"1137","product_id":"0071"}
+
+    """
+    def __init__(self, pci_addr, is_physical_function):
+        self.is_physical_function = is_physical_function
+        self._init_address_fields(pci_addr)
+
+    def _check_physical_function(self):
+        if self.pci_address_spec.is_single_address():
+            self.is_physical_function = (
+                utils.is_physical_function(
+                    self.pci_address_spec.domain,
+                    self.pci_address_spec.bus,
+                    self.pci_address_spec.slot,
+                    self.pci_address_spec.func))
+
+    def _init_address_fields(self, pci_addr):
+        if not self.is_physical_function:
+            if isinstance(pci_addr, six.string_types):
+                self.pci_address_spec = PciAddressGlobSpec(pci_addr)
+            elif isinstance(pci_addr, dict):
+                self.pci_address_spec = PciAddressRegexSpec(pci_addr)
+            else:
+                raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
+            self._check_physical_function()
+        else:
+            self.pci_address_spec = PhysicalPciAddress(pci_addr)
+
+    def match(self, pci_addr, pci_phys_addr):
+        """Match a device to this PciAddress.  Assume this is called given
+        pci_addr and pci_phys_addr reported by libvirt, no attempt is made to
+        verify if pci_addr is a VF of pci_phys_addr.
+
+        :param pci_addr: PCI address of the device to match.
+        :param pci_phys_addr: PCI address of the parent of the device to match
+                              (or None if the device is not a VF).
+        """
+
+        # Try to match on the parent PCI address if the PciDeviceSpec is a
+        # PF (sriov is available) and the device to match is a VF.  This
+        # makes it possible to specify the PCI address of a PF in the
+        # pci.passthrough_whitelist to match any of its VFs' PCI addresses.
+        if self.is_physical_function and pci_phys_addr:
+            pci_phys_addr_obj = PhysicalPciAddress(pci_phys_addr)
+            if self.pci_address_spec.match(pci_phys_addr_obj):
+                return True
+
+        # Try to match on the device PCI address only.
+        pci_addr_obj = PhysicalPciAddress(pci_addr)
+        return self.pci_address_spec.match(pci_addr_obj)
+
+
+class PciDeviceSpec(PciAddressSpec):
+    def __init__(self, dev_spec):
+        self.tags = dev_spec
+        self._init_dev_details()
+
+    def _init_dev_details(self):
+        self.vendor_id = self.tags.pop("vendor_id", ANY)
+        self.product_id = self.tags.pop("product_id", ANY)
+        # Note(moshele): The address attribute can be a string or a dict.
+        # For glob syntax or specific pci it is a string and for regex syntax
+        # it is a dict. The WhitelistPciAddress class handles both types.
+        self.address = self.tags.pop("address", None)
+        self.dev_name = self.tags.pop("devname", None)
+
+        self.vendor_id = self.vendor_id.strip()
+        self._set_pci_dev_info('vendor_id', MAX_VENDOR_ID, '%04x')
+        self._set_pci_dev_info('product_id', MAX_PRODUCT_ID, '%04x')
+
+        if self.address and self.dev_name:
+            raise exception.PciDeviceInvalidDeviceName()
+        if not self.dev_name:
+            pci_address = self.address or "*:*:*.*"
+            self.address = WhitelistPciAddress(pci_address, False)
+
+    def match(self, dev_dict):
+        if self.dev_name:
+            address_str, pf = utils.get_function_by_ifname(
+                self.dev_name)
+            if not address_str:
+                return False
+            # Note(moshele): In this case we always passing a string
+            # of the PF pci address
+            address_obj = WhitelistPciAddress(address_str, pf)
+        elif self.address:
+            address_obj = self.address
+        return all([
+            self.vendor_id in (ANY, dev_dict['vendor_id']),
+            self.product_id in (ANY, dev_dict['product_id']),
+            address_obj.match(dev_dict['address'],
+                              dev_dict.get('parent_addr'))])
+
+    def match_pci_obj(self, pci_obj):
+        return self.match({'vendor_id': pci_obj.vendor_id,
+                           'product_id': pci_obj.product_id,
+                           'address': pci_obj.address,
+                           'parent_addr': pci_obj.parent_addr})
+
+    def get_tags(self):
+        return self.tags

cyborg/accelerator/drivers/pci/pci/driver.py

@@ -0,0 +1,30 @@
+# Copyright 2024 Inspur.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""
+Cyborg PCI driver implementation.
+"""
+
+from cyborg.accelerator.drivers.pci.base import PciDriver
+from cyborg.accelerator.drivers.pci.pci import sysinfo
+
+
+class PCIDriver(PciDriver):
+    """Class for Pci drivers.
+       Vendor should implement their specific drivers in this class.
+    """
+
+    def discover(self):
+        return sysinfo.discover()

cyborg/accelerator/drivers/pci/pci/sysinfo.py

@@ -0,0 +1,147 @@
+# Copyright 2024 Inspur.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+"""
+Cyborg PCI driver implementation.
+"""
+from oslo_log import log as logging
+from oslo_serialization import jsonutils
+
+from cyborg.accelerator.common import utils
+from cyborg.accelerator.drivers.pci import utils as pci_utils
+from cyborg.accelerator.drivers.pci import whitelist
+from cyborg.common import constants
+import cyborg.conf
+from cyborg.objects.driver_objects import driver_attach_handle
+from cyborg.objects.driver_objects import driver_attribute
+from cyborg.objects.driver_objects import driver_controlpath_id
+from cyborg.objects.driver_objects import driver_deployable
+from cyborg.objects.driver_objects import driver_device
+
+LOG = logging.getLogger(__name__)
+CONF = cyborg.conf.CONF
+
+
+def _get_traits(vendor_id, product_id):
+    """Generate traits for PCIs.
+    : param vendor_id: vendor_id of PCI, eg."10de"
+    : param product_id: product_id of PCI, eg."1eb8".
+
+    Example PGPU traits:
+    {traits:["OWNER_CYBORG", "CUSTOM_PCI_1EB8"]}
+    """
+    vendor_name = pci_utils.VENDOR_MAPS.get(vendor_id).upper()
+    traits = ["CUSTOM_PCI_" + vendor_name]
+    # PCIE trait
+    product_trait = "_".join(('CUSTOM_PCI_PRODUCT_ID', product_id.upper()))
+    traits.append(product_trait)
+    return {"traits": traits}
+
+
+def _generate_attribute_list(pci):
+    attr_list = []
+    index = 0
+    for k, v in pci.items():
+        if k == "rc":
+            driver_attr = driver_attribute.DriverAttribute()
+            driver_attr.key, driver_attr.value = k, v
+            attr_list.append(driver_attr)
+        if k == "traits":
+            values = pci.get(k, [])
+            for val in values:
+                driver_attr = driver_attribute.DriverAttribute(
+                    key="trait" + str(index), value=val)
+                index = index + 1
+                attr_list.append(driver_attr)
+    return attr_list
+
+
+def _generate_attach_handle(pci):
+    driver_ah = driver_attach_handle.DriverAttachHandle()
+    driver_ah.in_use = False
+    driver_ah.attach_type = constants.AH_TYPE_PCI
+    driver_ah.attach_info = utils.pci_str_to_json(pci["devices"])
+    return driver_ah
+
+
+def _generate_dep_list(pci):
+    dep_list = []
+    driver_dep = driver_deployable.DriverDeployable()
+    driver_dep.attribute_list = _generate_attribute_list(pci)
+    driver_dep.attach_handle_list = []
+    # NOTE(wangzhh): The name of deployable should be unique, its format is
+    # under disscussion, may looks like
+    # <ComputeNodeName>_<NumaNodeName>_<CyborgName>_<NumInHost>
+    # NOTE(yumeng) Since Wallaby release, the deplpyable_name is named as
+    # <Compute_hostname>_<Device_address>
+    driver_dep.name = pci.get('hostname', '') + '_' + pci["devices"]
+    driver_dep.driver_name = \
+        pci_utils.VENDOR_MAPS.get(pci["vendor_id"]).upper()
+    driver_dep.num_accelerators = 1
+    driver_dep.attach_handle_list = [_generate_attach_handle(pci)]
+    dep_list.append(driver_dep)
+    return dep_list, driver_dep.num_accelerators
+
+
+def _generate_controlpath_id(pci):
+    driver_cpid = driver_controlpath_id.DriverControlPathID()
+    driver_cpid.cpid_type = "PCI"
+    driver_cpid.cpid_info = utils.pci_str_to_json(pci["devices"])
+    return driver_cpid
+
+
+def _generate_driver_device(pci):
+    driver_device_obj = driver_device.DriverDevice()
+    driver_device_obj.vendor = pci['vendor_id']
+    driver_device_obj.model = pci['product_id']
+    std_board_info = {'product_id': pci.get('product_id'),
+                      'controller': pci.get('controller'),
+                      }
+    driver_device_obj.std_board_info = jsonutils.dumps(std_board_info)
+    driver_device_obj.type = constants.DEVICE_GPU
+    driver_device_obj.stub = pci.get('stub', False)
+    driver_device_obj.controlpath_id = _generate_controlpath_id(pci)
+    driver_device_obj.deployable_list, ais = _generate_dep_list(pci)
+    driver_device_obj.vendor_board_info = pci.get('vendor_board_info',
+                                                  "miss_vb_info")
+    return driver_device_obj
+
+
+def _discover_pcis():
+    cyborg.conf.devices.register_dynamic_opts(CONF)
+    # discover pci devices by "lspci"
+    pci_list = []
+    pcis = pci_utils.get_pci_devices()
+    LOG.info('pcis:%s', pcis)
+    # report trait,rc and generate driver object
+    dev_filter = whitelist.Whitelist(CONF.pci.passthrough_whitelist)
+    for pci in pcis:
+        m = dev_filter.device_assignable(pci)
+        if m:
+            pci_dict = m.groupdict()
+            # get hostname for deployable_name usage
+            pci_dict['hostname'] = CONF.host
+            pci_dict["rc"] = constants.RESOURCES["PCI"]
+            traits = _get_traits(pci_dict["vendor_id"],
+                                 pci_dict["product_id"])
+            pci_dict.update(traits)
+            pci_list.append(_generate_driver_device(pci_dict))
+    LOG.info('pci_list:%s', pci_list)
+    return pci_list
+
+
+def discover():
+    devs = _discover_pcis()
+    return devs

cyborg/accelerator/drivers/pci/utils.py

@@ -0,0 +1,232 @@
+# Copyright 2024 Inspur.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import glob
+import os
+import re
+
+from oslo_concurrency import processutils
+from oslo_log import log as logging
+import six
+
+from cyborg.common import exception
+import cyborg.privsep
+
+LOG = logging.getLogger(__name__)
+
+PCI_VENDOR_PATTERN = "^(hex{4})$".replace("hex", r"[\da-fA-F]")
+_PCI_ADDRESS_PATTERN = ("^(hex{4}):(hex{2}):(hex{2}).(oct{1})$".
+                        replace("hex", r"[\da-fA-F]").
+                        replace("oct", "[0-7]"))
+_PCI_ADDRESS_REGEX = re.compile(_PCI_ADDRESS_PATTERN)
+
+_SRIOV_TOTALVFS = "sriov_totalvfs"
+
+VENDOR_MAPS = {"10de": "nvidia", "102b": "matrox"}
+
+
+@cyborg.privsep.sys_admin_pctxt.entrypoint
+def get_pci_devices():
+    cmd = ['lspci', '-nnn', '-D']
+    return processutils.execute(*cmd)
+
+
+def pci_device_prop_match(pci_dev, specs):
+    """Check if the pci_dev meet spec requirement
+
+    Specs is a list of PCI device property requirements.
+    An example of device requirement that the PCI should be either:
+    a) Device with vendor_id as 0x8086 and product_id as 0x8259, or
+    b) Device with vendor_id as 0x10de and product_id as 0x10d8:
+
+    [{"vendor_id":"8086", "product_id":"8259"},
+     {"vendor_id":"10de", "product_id":"10d8",
+      "capabilities_network": ["rx", "tx", "tso", "gso"]}]
+
+    """
+    def _matching_devices(spec):
+        for k, v in spec.items():
+            pci_dev_v = pci_dev.get(k)
+            if isinstance(v, list) and isinstance(pci_dev_v, list):
+                if not all(x in pci_dev.get(k) for x in v):
+                    return False
+            else:
+                # We don't need to check case for tags in order to avoid any
+                # mismatch with the tags provided by users for port
+                # binding profile and the ones configured by operators
+                # with pci whitelist option.
+                if isinstance(v, six.string_types):
+                    v = v.lower()
+                if isinstance(pci_dev_v, six.string_types):
+                    pci_dev_v = pci_dev_v.lower()
+                if pci_dev_v != v:
+                    return False
+        return True
+
+    return any(_matching_devices(spec) for spec in specs)
+
+
+def parse_address(address):
+    """Returns (domain, bus, slot, function) from PCI address that is stored in
+    PciDevice DB table.
+    """
+    m = _PCI_ADDRESS_REGEX.match(address)
+    if not m:
+        raise exception.PciDeviceWrongAddressFormat(address=address)
+    return m.groups()
+
+
+def get_pci_address_fields(pci_addr):
+    """Parse a fully-specified PCI device address.
+
+    Does not validate that the components are valid hex or wildcard values.
+
+    :param pci_addr: A string of the form "<domain>:<bus>:<slot>.<function>".
+    :return: A 4-tuple of strings ("<domain>", "<bus>", "<slot>", "<function>")
+    """
+    dbs, sep, func = pci_addr.partition('.')
+    domain, bus, slot = dbs.split(':')
+    return domain, bus, slot, func
+
+
+def get_pci_address(domain, bus, slot, func):
+    """Assembles PCI address components into a fully-specified PCI address.
+
+    Does not validate that the components are valid hex or wildcard values.
+
+    :param domain, bus, slot, func: Hex or wildcard strings.
+    :return: A string of the form "<domain>:<bus>:<slot>.<function>".
+    """
+    return '%s:%s:%s.%s' % (domain, bus, slot, func)
+
+
+def get_function_by_ifname(ifname):
+    """Given the device name, returns the PCI address of a device
+    and returns True if the address is in a physical function.
+    """
+    dev_path = "/sys/class/net/%s/device" % ifname
+    if os.path.isdir(dev_path):
+        try:
+            # sriov_totalvfs contains the maximum possible VFs for this PF
+            with open(os.path.join(dev_path, _SRIOV_TOTALVFS)) as fd:
+                sriov_totalvfs = int(fd.read())
+                return (os.readlink(dev_path).strip("./"),
+                        sriov_totalvfs > 0)
+        except (IOError, ValueError):
+            return os.readlink(dev_path).strip("./"), False
+    return None, False
+
+
+def is_physical_function(domain, bus, slot, function):
+    dev_path = "/sys/bus/pci/devices/%(d)s:%(b)s:%(s)s.%(f)s/" % {
+        "d": domain, "b": bus, "s": slot, "f": function}
+    if os.path.isdir(dev_path):
+        try:
+            with open(dev_path + _SRIOV_TOTALVFS) as fd:
+                sriov_totalvfs = int(fd.read())
+                return sriov_totalvfs > 0
+        except (IOError, ValueError):
+            pass
+    return False
+
+
+def _get_sysfs_netdev_path(pci_addr, pf_interface):
+    """Get the sysfs path based on the PCI address of the device.
+
+    Assumes a networking device - will not check for the existence of the path.
+    """
+    if pf_interface:
+        return "/sys/bus/pci/devices/%s/physfn/net" % pci_addr
+    return "/sys/bus/pci/devices/%s/net" % pci_addr
+
+
+def get_ifname_by_pci_address(pci_addr, pf_interface=False):
+    """Get the interface name based on a VF's pci address.
+
+    The returned interface name is either the parent PF's or that of the VF
+    itself based on the argument of pf_interface.
+    """
+    dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface)
+    try:
+        dev_info = os.listdir(dev_path)
+        return dev_info.pop()
+    except Exception:
+        raise exception.PciDeviceNotFoundById(id=pci_addr)
+
+
+def get_mac_by_pci_address(pci_addr, pf_interface=False):
+    """Get the MAC address of the nic based on its PCI address.
+
+    Raises PciDeviceNotFoundById in case the pci device is not a NIC
+    """
+    dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface)
+    if_name = get_ifname_by_pci_address(pci_addr, pf_interface)
+    addr_file = os.path.join(dev_path, if_name, 'address')
+
+    try:
+        with open(addr_file) as f:
+            mac = next(f).strip()
+            return mac
+    except (IOError, StopIteration) as e:
+        LOG.warning("Could not find the expected sysfs file for "
+                    "determining the MAC address of the PCI device "
+                    "%(addr)s. May not be a NIC. Error: %(e)s",
+                    {'addr': pci_addr, 'e': e})
+        raise exception.PciDeviceNotFoundById(id=pci_addr)
+
+
+def get_vf_num_by_pci_address(pci_addr):
+    """Get the VF number based on a VF's pci address
+
+    A VF is associated with an VF number, which ip link command uses to
+    configure it. This number can be obtained from the PCI device filesystem.
+    """
+    VIRTFN_RE = re.compile(r"virtfn(\d+)")
+    virtfns_path = "/sys/bus/pci/devices/%s/physfn/virtfn*" % (pci_addr)
+    vf_num = None
+    try:
+        for vf_path in glob.iglob(virtfns_path):
+            if re.search(pci_addr, os.readlink(vf_path)):
+                t = VIRTFN_RE.search(vf_path)
+                vf_num = t.group(1)
+                break
+    except Exception:
+        pass
+    if vf_num is None:
+        raise exception.PciDeviceNotFoundById(id=pci_addr)
+    return vf_num
+
+
+def get_net_name_by_vf_pci_address(vfaddress):
+    """Given the VF PCI address, returns the net device name.
+
+    Every VF is associated to a PCI network device. This function
+    returns the libvirt name given to this network device; e.g.:
+
+        <device>
+            <name>net_enp8s0f0_90_e2_ba_5e_a6_40</name>
+        ...
+
+    In the libvirt parser information tree, the network device stores the
+    network capabilities associated to this device.
+    """
+    try:
+        mac = get_mac_by_pci_address(vfaddress).split(':')
+        ifname = get_ifname_by_pci_address(vfaddress)
+        return ("net_%(ifname)s_%(mac)s" %
+                {'ifname': ifname, 'mac': '_'.join(mac)})
+    except Exception:
+        LOG.warning("No net device was found for VF %(vfaddress)s",
+                    {'vfaddress': vfaddress})
+        return

cyborg/accelerator/drivers/pci/whitelist.py

@@ -0,0 +1,91 @@
+# Copyright 2024 Inspur.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_serialization import jsonutils
+
+from cyborg.accelerator.drivers.pci import devspec
+from cyborg.common import exception
+from cyborg.common.i18n import _
+
+
+class Whitelist(object):
+    """White list class to represent assignable pci devices.
+
+    Not all devices on a compute node can be assigned to a guest. The cloud
+    administrator decides which devices can be assigned based on ``vendor_id``
+    or ``product_id``, etc. If no white list is specified, no devices will be
+    assignable.
+    """
+
+    def __init__(self, whitelist_spec=None):
+        """White list constructor
+
+        For example, the following json string specifies that devices whose
+        vendor_id is '8086' and product_id is '1520' can be assigned
+        to guests. ::
+
+            '[{"product_id":"1520", "vendor_id":"8086"}]'
+
+        :param whitelist_spec: A JSON string for a dictionary or list thereof.
+            Each dictionary specifies the pci device properties requirement.
+            See the definition of ``passthrough_whitelist`` in
+            ``nova.conf.pci`` for details and examples.
+        """
+        if whitelist_spec:
+            self.specs = self._parse_white_list_from_config(whitelist_spec)
+        else:
+            self.specs = []
+
+    @staticmethod
+    def _parse_white_list_from_config(whitelists):
+        """Parse and validate the pci whitelist from the nova config."""
+        specs = []
+        for jsonspec in whitelists:
+            try:
+                dev_spec = jsonutils.loads(jsonspec)
+            except ValueError:
+                raise exception.PciConfigInvalidWhitelist(
+                    reason=_("Invalid entry: '%s'") % jsonspec)
+            if isinstance(dev_spec, dict):
+                dev_spec = [dev_spec]
+            elif not isinstance(dev_spec, list):
+                raise exception.PciConfigInvalidWhitelist(
+                    reason=_("Invalid entry: '%s'; "
+                             "Expecting list or dict") % jsonspec)
+
+            for ds in dev_spec:
+                if not isinstance(ds, dict):
+                    raise exception.PciConfigInvalidWhitelist(
+                        reason=_("Invalid entry: '%s'; "
+                                 "Expecting dict") % ds)
+
+                spec = devspec.PciDeviceSpec(ds)
+                specs.append(spec)
+
+        return specs
+
+    def device_assignable(self, dev):
+        """Check if a device can be assigned to a guest.
+
+        :param dev: A dictionary describing the device properties
+        """
+        for spec in self.specs:
+            if spec.match(dev):
+                return True
+        return False
+
+    def get_devspec(self, pci_dev):
+        for spec in self.specs:
+            if spec.match_pci_obj(pci_dev):
+                return spec

cyborg/common/constants.py

@@ -88,6 +88,7 @@ RESOURCES = {
     "QAT": "CUSTOM_QAT",
     "NIC": "CUSTOM_NIC",
     "SSD": 'CUSTOM_SSD',
+    "PCI": 'CUSTOM_PCI',
 }
 
 

cyborg/common/exception.py

@@ -425,3 +425,12 @@ class FPGAProgramError(CyborgException):
 
 class PciDeviceNotFoundById(NotFound):
     _msg_fmt = _("PCI device %(id)s not found")
+
+
+class PciConfigInvalidWhitelist(Invalid):
+    _msg_fmt = _("Invalid PCI devices whitelist config: %(reason)s.")
+
+
+class PciDeviceInvalidDeviceName(CyborgException):
+    _msg_fmt = _("Invalid PCI whitelist: The PCI whitelist can specify "
+                 "devname or address, but not both.")

cyborg/conf/devices.py

@@ -14,6 +14,15 @@
 
 from oslo_config import cfg
 
+pci_group = cfg.OptGroup(
+    name='pci',
+    title='PCI passthrough options')
+
+pci_opts = [
+    cfg.MultiStrOpt('passthrough_whitelist',
+                    default=[],
+                    help=" ")
+]
 
 nic_group = cfg.OptGroup(
     name='nic_devices',
@@ -72,6 +81,8 @@ def register_opts(conf):
     conf.register_opts(nic_opts, group=nic_group)
     conf.register_group(gpu_group)
     conf.register_opts(vgpu_opts, group=gpu_group)
+    conf.register_group(pci_group)
+    conf.register_opts(pci_opts, group=pci_group)
 
 
 def register_dynamic_opts(conf):

setup.cfg

@@ -57,6 +57,7 @@ cyborg.accelerator.driver =
     intel_qat_driver = cyborg.accelerator.drivers.qat.intel.driver:IntelQATDriver
     intel_nic_driver = cyborg.accelerator.drivers.nic.intel.driver:IntelNICDriver
     inspur_nvme_ssd_driver = cyborg.accelerator.drivers.ssd.inspur.driver:InspurNVMeSSDDriver
+    pci_driver = cyborg.accelerator.drivers.pci.pci.driver:PCIDriver
 
 oslo.config.opts =
     cyborg = cyborg.conf.opts:list_opts