From 5f896d2931d1a02e4beaf2cb4109a8a3e384b155 Mon Sep 17 00:00:00 2001 From: songwenping Date: Fri, 28 Jun 2024 17:10:31 +0800 Subject: [PATCH] pci unify management bp pcie-unified-management Change-Id: I2a285b61c4f41e999670715dfa2cef8c910467b9 Change-Id: Iaddcd2d5dfef987b3d11d30a76e2775c8e60ba38 --- cyborg/accelerator/drivers/pci/__init__.py | 0 cyborg/accelerator/drivers/pci/base.py | 29 ++ cyborg/accelerator/drivers/pci/devspec.py | 290 ++++++++++++++++++ .../accelerator/drivers/pci/pci/__init__.py | 0 cyborg/accelerator/drivers/pci/pci/driver.py | 30 ++ cyborg/accelerator/drivers/pci/pci/sysinfo.py | 147 +++++++++ cyborg/accelerator/drivers/pci/utils.py | 232 ++++++++++++++ cyborg/accelerator/drivers/pci/whitelist.py | 91 ++++++ cyborg/common/constants.py | 1 + cyborg/common/exception.py | 9 + cyborg/conf/devices.py | 11 + setup.cfg | 1 + 12 files changed, 841 insertions(+) create mode 100644 cyborg/accelerator/drivers/pci/__init__.py create mode 100644 cyborg/accelerator/drivers/pci/base.py create mode 100644 cyborg/accelerator/drivers/pci/devspec.py create mode 100644 cyborg/accelerator/drivers/pci/pci/__init__.py create mode 100644 cyborg/accelerator/drivers/pci/pci/driver.py create mode 100644 cyborg/accelerator/drivers/pci/pci/sysinfo.py create mode 100644 cyborg/accelerator/drivers/pci/utils.py create mode 100644 cyborg/accelerator/drivers/pci/whitelist.py diff --git a/cyborg/accelerator/drivers/pci/__init__.py b/cyborg/accelerator/drivers/pci/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/accelerator/drivers/pci/base.py b/cyborg/accelerator/drivers/pci/base.py new file mode 100644 index 00000000..9aa52f1c --- /dev/null +++ b/cyborg/accelerator/drivers/pci/base.py @@ -0,0 +1,29 @@ +# Copyright 2024 Inspur. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg Pci driver implementation. +""" + + +class PciDriver(object): + """Base class for Pci drivers. + """ + + def __init__(self, *args, **kwargs): + pass + + def discover(self): + raise NotImplementedError() diff --git a/cyborg/accelerator/drivers/pci/devspec.py b/cyborg/accelerator/drivers/pci/devspec.py new file mode 100644 index 00000000..0e61ac07 --- /dev/null +++ b/cyborg/accelerator/drivers/pci/devspec.py @@ -0,0 +1,290 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import abc +import re +import string + +import six + +from cyborg.accelerator.drivers.pci import utils +from cyborg.common import exception +from cyborg.common.i18n import _ + +MAX_VENDOR_ID = 0xFFFF +MAX_PRODUCT_ID = 0xFFFF +MAX_FUNC = 0x7 +MAX_DOMAIN = 0xFFFF +MAX_BUS = 0xFF +MAX_SLOT = 0x1F +ANY = '*' +REGEX_ANY = '.*' + + +@six.add_metaclass(abc.ABCMeta) +class PciAddressSpec(object): + """Abstract class for all PCI address spec styles + + This class checks the address fields of the pci.passthrough_whitelist + """ + + @abc.abstractmethod + def match(self, pci_addr): + pass + + def is_single_address(self): + return all([ + all(c in string.hexdigits for c in self.domain), + all(c in string.hexdigits for c in self.bus), + all(c in string.hexdigits for c in self.slot), + all(c in string.hexdigits for c in self.func)]) + + def _set_pci_dev_info(self, prop, maxval, hex_value): + a = getattr(self, prop) + if a == ANY: + return + try: + v = int(a, 16) + except ValueError: + raise exception.PciConfigInvalidWhitelist( + reason=_("property %(property)s ('%(attr)s') does not parse " + "as a hex number.") % {'property': prop, 'attr': a}) + if v > maxval: + raise exception.PciConfigInvalidWhitelist( + reason=_("property %(property)s (%(attr)s) is greater than " + "the maximum allowable value (%(max)X).") % { + 'property': prop, 'attr': a, 'max': maxval}) + setattr(self, prop, hex_value % v) + + +class PhysicalPciAddress(PciAddressSpec): + """Manages the address fields for a fully-qualified PCI address. + + This function class will validate the address fields for a single + PCI device. + """ + def __init__(self, pci_addr): + try: + if isinstance(pci_addr, dict): + self.domain = pci_addr['domain'] + self.bus = pci_addr['bus'] + self.slot = pci_addr['slot'] + self.func = pci_addr['function'] + else: + self.domain, self.bus, self.slot, self.func = ( + utils.get_pci_address_fields(pci_addr)) + self._set_pci_dev_info('func', MAX_FUNC, '%1x') + self._set_pci_dev_info('domain', MAX_DOMAIN, '%04x') + self._set_pci_dev_info('bus', MAX_BUS, '%02x') + self._set_pci_dev_info('slot', MAX_SLOT, '%02x') + except (KeyError, ValueError): + raise exception.PciDeviceWrongAddressFormat(address=pci_addr) + + def match(self, phys_pci_addr): + conditions = [ + self.domain == phys_pci_addr.domain, + self.bus == phys_pci_addr.bus, + self.slot == phys_pci_addr.slot, + self.func == phys_pci_addr.func, + ] + return all(conditions) + + +class PciAddressGlobSpec(PciAddressSpec): + """Manages the address fields with glob style. + + This function class will validate the address fields with glob style, + check for wildcards, and insert wildcards where the field is left blank. + """ + + def __init__(self, pci_addr): + self.domain = ANY + self.bus = ANY + self.slot = ANY + self.func = ANY + + dbs, sep, func = pci_addr.partition('.') + if func: + self.func = func.strip() + self._set_pci_dev_info('func', MAX_FUNC, '%01x') + if dbs: + dbs_fields = dbs.split(':') + if len(dbs_fields) > 3: + raise exception.PciDeviceWrongAddressFormat(address=pci_addr) + # If we got a partial address like ":00.", we need to turn this + # into a domain of ANY, a bus of ANY, and a slot of 00. This code + # allows the address bus and/or domain to be left off + dbs_all = [ANY] * (3 - len(dbs_fields)) + dbs_all.extend(dbs_fields) + dbs_checked = [s.strip() or ANY for s in dbs_all] + self.domain, self.bus, self.slot = dbs_checked + self._set_pci_dev_info('domain', MAX_DOMAIN, '%04x') + self._set_pci_dev_info('bus', MAX_BUS, '%02x') + self._set_pci_dev_info('slot', MAX_SLOT, '%02x') + + def match(self, phys_pci_addr): + conditions = [ + self.domain in (ANY, phys_pci_addr.domain), + self.bus in (ANY, phys_pci_addr.bus), + self.slot in (ANY, phys_pci_addr.slot), + self.func in (ANY, phys_pci_addr.func) + ] + return all(conditions) + + +class PciAddressRegexSpec(PciAddressSpec): + """Manages the address fields with regex style. + + This function class will validate the address fields with regex style. + The validation includes check for all PCI address attributes and validate + their regex. + """ + def __init__(self, pci_addr): + try: + self.domain = pci_addr.get('domain', REGEX_ANY) + self.bus = pci_addr.get('bus', REGEX_ANY) + self.slot = pci_addr.get('slot', REGEX_ANY) + self.func = pci_addr.get('function', REGEX_ANY) + self.domain_regex = re.compile(self.domain) + self.bus_regex = re.compile(self.bus) + self.slot_regex = re.compile(self.slot) + self.func_regex = re.compile(self.func) + except re.error: + raise exception.PciDeviceWrongAddressFormat(address=pci_addr) + + def match(self, phys_pci_addr): + conditions = [ + bool(self.domain_regex.match(phys_pci_addr.domain)), + bool(self.bus_regex.match(phys_pci_addr.bus)), + bool(self.slot_regex.match(phys_pci_addr.slot)), + bool(self.func_regex.match(phys_pci_addr.func)) + ] + return all(conditions) + + +class WhitelistPciAddress(object): + """Manages the address fields of the whitelist. + + This class checks the address fields of the pci.passthrough_whitelist + configuration option, validating the address fields. + Example configs: + + | [pci] + | passthrough_whitelist = {"address":"*:0a:00.*", + | "physical_network":"physnet1"} + | passthrough_whitelist = {"address": {"domain": ".*", + "bus": "02", + "slot": "01", + "function": "[0-2]"}, + "physical_network":"net1"} + | passthrough_whitelist = {"vendor_id":"1137","product_id":"0071"} + + """ + def __init__(self, pci_addr, is_physical_function): + self.is_physical_function = is_physical_function + self._init_address_fields(pci_addr) + + def _check_physical_function(self): + if self.pci_address_spec.is_single_address(): + self.is_physical_function = ( + utils.is_physical_function( + self.pci_address_spec.domain, + self.pci_address_spec.bus, + self.pci_address_spec.slot, + self.pci_address_spec.func)) + + def _init_address_fields(self, pci_addr): + if not self.is_physical_function: + if isinstance(pci_addr, six.string_types): + self.pci_address_spec = PciAddressGlobSpec(pci_addr) + elif isinstance(pci_addr, dict): + self.pci_address_spec = PciAddressRegexSpec(pci_addr) + else: + raise exception.PciDeviceWrongAddressFormat(address=pci_addr) + self._check_physical_function() + else: + self.pci_address_spec = PhysicalPciAddress(pci_addr) + + def match(self, pci_addr, pci_phys_addr): + """Match a device to this PciAddress. Assume this is called given + pci_addr and pci_phys_addr reported by libvirt, no attempt is made to + verify if pci_addr is a VF of pci_phys_addr. + + :param pci_addr: PCI address of the device to match. + :param pci_phys_addr: PCI address of the parent of the device to match + (or None if the device is not a VF). + """ + + # Try to match on the parent PCI address if the PciDeviceSpec is a + # PF (sriov is available) and the device to match is a VF. This + # makes it possible to specify the PCI address of a PF in the + # pci.passthrough_whitelist to match any of its VFs' PCI addresses. + if self.is_physical_function and pci_phys_addr: + pci_phys_addr_obj = PhysicalPciAddress(pci_phys_addr) + if self.pci_address_spec.match(pci_phys_addr_obj): + return True + + # Try to match on the device PCI address only. + pci_addr_obj = PhysicalPciAddress(pci_addr) + return self.pci_address_spec.match(pci_addr_obj) + + +class PciDeviceSpec(PciAddressSpec): + def __init__(self, dev_spec): + self.tags = dev_spec + self._init_dev_details() + + def _init_dev_details(self): + self.vendor_id = self.tags.pop("vendor_id", ANY) + self.product_id = self.tags.pop("product_id", ANY) + # Note(moshele): The address attribute can be a string or a dict. + # For glob syntax or specific pci it is a string and for regex syntax + # it is a dict. The WhitelistPciAddress class handles both types. + self.address = self.tags.pop("address", None) + self.dev_name = self.tags.pop("devname", None) + + self.vendor_id = self.vendor_id.strip() + self._set_pci_dev_info('vendor_id', MAX_VENDOR_ID, '%04x') + self._set_pci_dev_info('product_id', MAX_PRODUCT_ID, '%04x') + + if self.address and self.dev_name: + raise exception.PciDeviceInvalidDeviceName() + if not self.dev_name: + pci_address = self.address or "*:*:*.*" + self.address = WhitelistPciAddress(pci_address, False) + + def match(self, dev_dict): + if self.dev_name: + address_str, pf = utils.get_function_by_ifname( + self.dev_name) + if not address_str: + return False + # Note(moshele): In this case we always passing a string + # of the PF pci address + address_obj = WhitelistPciAddress(address_str, pf) + elif self.address: + address_obj = self.address + return all([ + self.vendor_id in (ANY, dev_dict['vendor_id']), + self.product_id in (ANY, dev_dict['product_id']), + address_obj.match(dev_dict['address'], + dev_dict.get('parent_addr'))]) + + def match_pci_obj(self, pci_obj): + return self.match({'vendor_id': pci_obj.vendor_id, + 'product_id': pci_obj.product_id, + 'address': pci_obj.address, + 'parent_addr': pci_obj.parent_addr}) + + def get_tags(self): + return self.tags diff --git a/cyborg/accelerator/drivers/pci/pci/__init__.py b/cyborg/accelerator/drivers/pci/pci/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/accelerator/drivers/pci/pci/driver.py b/cyborg/accelerator/drivers/pci/pci/driver.py new file mode 100644 index 00000000..c77e1196 --- /dev/null +++ b/cyborg/accelerator/drivers/pci/pci/driver.py @@ -0,0 +1,30 @@ +# Copyright 2024 Inspur. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg PCI driver implementation. +""" + +from cyborg.accelerator.drivers.pci.base import PciDriver +from cyborg.accelerator.drivers.pci.pci import sysinfo + + +class PCIDriver(PciDriver): + """Class for Pci drivers. + Vendor should implement their specific drivers in this class. + """ + + def discover(self): + return sysinfo.discover() diff --git a/cyborg/accelerator/drivers/pci/pci/sysinfo.py b/cyborg/accelerator/drivers/pci/pci/sysinfo.py new file mode 100644 index 00000000..e48337f3 --- /dev/null +++ b/cyborg/accelerator/drivers/pci/pci/sysinfo.py @@ -0,0 +1,147 @@ +# Copyright 2024 Inspur. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg PCI driver implementation. +""" +from oslo_log import log as logging +from oslo_serialization import jsonutils + +from cyborg.accelerator.common import utils +from cyborg.accelerator.drivers.pci import utils as pci_utils +from cyborg.accelerator.drivers.pci import whitelist +from cyborg.common import constants +import cyborg.conf +from cyborg.objects.driver_objects import driver_attach_handle +from cyborg.objects.driver_objects import driver_attribute +from cyborg.objects.driver_objects import driver_controlpath_id +from cyborg.objects.driver_objects import driver_deployable +from cyborg.objects.driver_objects import driver_device + +LOG = logging.getLogger(__name__) +CONF = cyborg.conf.CONF + + +def _get_traits(vendor_id, product_id): + """Generate traits for PCIs. + : param vendor_id: vendor_id of PCI, eg."10de" + : param product_id: product_id of PCI, eg."1eb8". + + Example PGPU traits: + {traits:["OWNER_CYBORG", "CUSTOM_PCI_1EB8"]} + """ + vendor_name = pci_utils.VENDOR_MAPS.get(vendor_id).upper() + traits = ["CUSTOM_PCI_" + vendor_name] + # PCIE trait + product_trait = "_".join(('CUSTOM_PCI_PRODUCT_ID', product_id.upper())) + traits.append(product_trait) + return {"traits": traits} + + +def _generate_attribute_list(pci): + attr_list = [] + index = 0 + for k, v in pci.items(): + if k == "rc": + driver_attr = driver_attribute.DriverAttribute() + driver_attr.key, driver_attr.value = k, v + attr_list.append(driver_attr) + if k == "traits": + values = pci.get(k, []) + for val in values: + driver_attr = driver_attribute.DriverAttribute( + key="trait" + str(index), value=val) + index = index + 1 + attr_list.append(driver_attr) + return attr_list + + +def _generate_attach_handle(pci): + driver_ah = driver_attach_handle.DriverAttachHandle() + driver_ah.in_use = False + driver_ah.attach_type = constants.AH_TYPE_PCI + driver_ah.attach_info = utils.pci_str_to_json(pci["devices"]) + return driver_ah + + +def _generate_dep_list(pci): + dep_list = [] + driver_dep = driver_deployable.DriverDeployable() + driver_dep.attribute_list = _generate_attribute_list(pci) + driver_dep.attach_handle_list = [] + # NOTE(wangzhh): The name of deployable should be unique, its format is + # under disscussion, may looks like + # ___ + # NOTE(yumeng) Since Wallaby release, the deplpyable_name is named as + # _ + driver_dep.name = pci.get('hostname', '') + '_' + pci["devices"] + driver_dep.driver_name = \ + pci_utils.VENDOR_MAPS.get(pci["vendor_id"]).upper() + driver_dep.num_accelerators = 1 + driver_dep.attach_handle_list = [_generate_attach_handle(pci)] + dep_list.append(driver_dep) + return dep_list, driver_dep.num_accelerators + + +def _generate_controlpath_id(pci): + driver_cpid = driver_controlpath_id.DriverControlPathID() + driver_cpid.cpid_type = "PCI" + driver_cpid.cpid_info = utils.pci_str_to_json(pci["devices"]) + return driver_cpid + + +def _generate_driver_device(pci): + driver_device_obj = driver_device.DriverDevice() + driver_device_obj.vendor = pci['vendor_id'] + driver_device_obj.model = pci['product_id'] + std_board_info = {'product_id': pci.get('product_id'), + 'controller': pci.get('controller'), + } + driver_device_obj.std_board_info = jsonutils.dumps(std_board_info) + driver_device_obj.type = constants.DEVICE_GPU + driver_device_obj.stub = pci.get('stub', False) + driver_device_obj.controlpath_id = _generate_controlpath_id(pci) + driver_device_obj.deployable_list, ais = _generate_dep_list(pci) + driver_device_obj.vendor_board_info = pci.get('vendor_board_info', + "miss_vb_info") + return driver_device_obj + + +def _discover_pcis(): + cyborg.conf.devices.register_dynamic_opts(CONF) + # discover pci devices by "lspci" + pci_list = [] + pcis = pci_utils.get_pci_devices() + LOG.info('pcis:%s', pcis) + # report trait,rc and generate driver object + dev_filter = whitelist.Whitelist(CONF.pci.passthrough_whitelist) + for pci in pcis: + m = dev_filter.device_assignable(pci) + if m: + pci_dict = m.groupdict() + # get hostname for deployable_name usage + pci_dict['hostname'] = CONF.host + pci_dict["rc"] = constants.RESOURCES["PCI"] + traits = _get_traits(pci_dict["vendor_id"], + pci_dict["product_id"]) + pci_dict.update(traits) + pci_list.append(_generate_driver_device(pci_dict)) + LOG.info('pci_list:%s', pci_list) + return pci_list + + +def discover(): + devs = _discover_pcis() + return devs diff --git a/cyborg/accelerator/drivers/pci/utils.py b/cyborg/accelerator/drivers/pci/utils.py new file mode 100644 index 00000000..5ead8e5f --- /dev/null +++ b/cyborg/accelerator/drivers/pci/utils.py @@ -0,0 +1,232 @@ +# Copyright 2024 Inspur. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import glob +import os +import re + +from oslo_concurrency import processutils +from oslo_log import log as logging +import six + +from cyborg.common import exception +import cyborg.privsep + +LOG = logging.getLogger(__name__) + +PCI_VENDOR_PATTERN = "^(hex{4})$".replace("hex", r"[\da-fA-F]") +_PCI_ADDRESS_PATTERN = ("^(hex{4}):(hex{2}):(hex{2}).(oct{1})$". + replace("hex", r"[\da-fA-F]"). + replace("oct", "[0-7]")) +_PCI_ADDRESS_REGEX = re.compile(_PCI_ADDRESS_PATTERN) + +_SRIOV_TOTALVFS = "sriov_totalvfs" + +VENDOR_MAPS = {"10de": "nvidia", "102b": "matrox"} + + +@cyborg.privsep.sys_admin_pctxt.entrypoint +def get_pci_devices(): + cmd = ['lspci', '-nnn', '-D'] + return processutils.execute(*cmd) + + +def pci_device_prop_match(pci_dev, specs): + """Check if the pci_dev meet spec requirement + + Specs is a list of PCI device property requirements. + An example of device requirement that the PCI should be either: + a) Device with vendor_id as 0x8086 and product_id as 0x8259, or + b) Device with vendor_id as 0x10de and product_id as 0x10d8: + + [{"vendor_id":"8086", "product_id":"8259"}, + {"vendor_id":"10de", "product_id":"10d8", + "capabilities_network": ["rx", "tx", "tso", "gso"]}] + + """ + def _matching_devices(spec): + for k, v in spec.items(): + pci_dev_v = pci_dev.get(k) + if isinstance(v, list) and isinstance(pci_dev_v, list): + if not all(x in pci_dev.get(k) for x in v): + return False + else: + # We don't need to check case for tags in order to avoid any + # mismatch with the tags provided by users for port + # binding profile and the ones configured by operators + # with pci whitelist option. + if isinstance(v, six.string_types): + v = v.lower() + if isinstance(pci_dev_v, six.string_types): + pci_dev_v = pci_dev_v.lower() + if pci_dev_v != v: + return False + return True + + return any(_matching_devices(spec) for spec in specs) + + +def parse_address(address): + """Returns (domain, bus, slot, function) from PCI address that is stored in + PciDevice DB table. + """ + m = _PCI_ADDRESS_REGEX.match(address) + if not m: + raise exception.PciDeviceWrongAddressFormat(address=address) + return m.groups() + + +def get_pci_address_fields(pci_addr): + """Parse a fully-specified PCI device address. + + Does not validate that the components are valid hex or wildcard values. + + :param pci_addr: A string of the form "::.". + :return: A 4-tuple of strings ("", "", "", "") + """ + dbs, sep, func = pci_addr.partition('.') + domain, bus, slot = dbs.split(':') + return domain, bus, slot, func + + +def get_pci_address(domain, bus, slot, func): + """Assembles PCI address components into a fully-specified PCI address. + + Does not validate that the components are valid hex or wildcard values. + + :param domain, bus, slot, func: Hex or wildcard strings. + :return: A string of the form "::.". + """ + return '%s:%s:%s.%s' % (domain, bus, slot, func) + + +def get_function_by_ifname(ifname): + """Given the device name, returns the PCI address of a device + and returns True if the address is in a physical function. + """ + dev_path = "/sys/class/net/%s/device" % ifname + if os.path.isdir(dev_path): + try: + # sriov_totalvfs contains the maximum possible VFs for this PF + with open(os.path.join(dev_path, _SRIOV_TOTALVFS)) as fd: + sriov_totalvfs = int(fd.read()) + return (os.readlink(dev_path).strip("./"), + sriov_totalvfs > 0) + except (IOError, ValueError): + return os.readlink(dev_path).strip("./"), False + return None, False + + +def is_physical_function(domain, bus, slot, function): + dev_path = "/sys/bus/pci/devices/%(d)s:%(b)s:%(s)s.%(f)s/" % { + "d": domain, "b": bus, "s": slot, "f": function} + if os.path.isdir(dev_path): + try: + with open(dev_path + _SRIOV_TOTALVFS) as fd: + sriov_totalvfs = int(fd.read()) + return sriov_totalvfs > 0 + except (IOError, ValueError): + pass + return False + + +def _get_sysfs_netdev_path(pci_addr, pf_interface): + """Get the sysfs path based on the PCI address of the device. + + Assumes a networking device - will not check for the existence of the path. + """ + if pf_interface: + return "/sys/bus/pci/devices/%s/physfn/net" % pci_addr + return "/sys/bus/pci/devices/%s/net" % pci_addr + + +def get_ifname_by_pci_address(pci_addr, pf_interface=False): + """Get the interface name based on a VF's pci address. + + The returned interface name is either the parent PF's or that of the VF + itself based on the argument of pf_interface. + """ + dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface) + try: + dev_info = os.listdir(dev_path) + return dev_info.pop() + except Exception: + raise exception.PciDeviceNotFoundById(id=pci_addr) + + +def get_mac_by_pci_address(pci_addr, pf_interface=False): + """Get the MAC address of the nic based on its PCI address. + + Raises PciDeviceNotFoundById in case the pci device is not a NIC + """ + dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface) + if_name = get_ifname_by_pci_address(pci_addr, pf_interface) + addr_file = os.path.join(dev_path, if_name, 'address') + + try: + with open(addr_file) as f: + mac = next(f).strip() + return mac + except (IOError, StopIteration) as e: + LOG.warning("Could not find the expected sysfs file for " + "determining the MAC address of the PCI device " + "%(addr)s. May not be a NIC. Error: %(e)s", + {'addr': pci_addr, 'e': e}) + raise exception.PciDeviceNotFoundById(id=pci_addr) + + +def get_vf_num_by_pci_address(pci_addr): + """Get the VF number based on a VF's pci address + + A VF is associated with an VF number, which ip link command uses to + configure it. This number can be obtained from the PCI device filesystem. + """ + VIRTFN_RE = re.compile(r"virtfn(\d+)") + virtfns_path = "/sys/bus/pci/devices/%s/physfn/virtfn*" % (pci_addr) + vf_num = None + try: + for vf_path in glob.iglob(virtfns_path): + if re.search(pci_addr, os.readlink(vf_path)): + t = VIRTFN_RE.search(vf_path) + vf_num = t.group(1) + break + except Exception: + pass + if vf_num is None: + raise exception.PciDeviceNotFoundById(id=pci_addr) + return vf_num + + +def get_net_name_by_vf_pci_address(vfaddress): + """Given the VF PCI address, returns the net device name. + + Every VF is associated to a PCI network device. This function + returns the libvirt name given to this network device; e.g.: + + + net_enp8s0f0_90_e2_ba_5e_a6_40 + ... + + In the libvirt parser information tree, the network device stores the + network capabilities associated to this device. + """ + try: + mac = get_mac_by_pci_address(vfaddress).split(':') + ifname = get_ifname_by_pci_address(vfaddress) + return ("net_%(ifname)s_%(mac)s" % + {'ifname': ifname, 'mac': '_'.join(mac)}) + except Exception: + LOG.warning("No net device was found for VF %(vfaddress)s", + {'vfaddress': vfaddress}) + return diff --git a/cyborg/accelerator/drivers/pci/whitelist.py b/cyborg/accelerator/drivers/pci/whitelist.py new file mode 100644 index 00000000..54dde9d2 --- /dev/null +++ b/cyborg/accelerator/drivers/pci/whitelist.py @@ -0,0 +1,91 @@ +# Copyright 2024 Inspur. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils + +from cyborg.accelerator.drivers.pci import devspec +from cyborg.common import exception +from cyborg.common.i18n import _ + + +class Whitelist(object): + """White list class to represent assignable pci devices. + + Not all devices on a compute node can be assigned to a guest. The cloud + administrator decides which devices can be assigned based on ``vendor_id`` + or ``product_id``, etc. If no white list is specified, no devices will be + assignable. + """ + + def __init__(self, whitelist_spec=None): + """White list constructor + + For example, the following json string specifies that devices whose + vendor_id is '8086' and product_id is '1520' can be assigned + to guests. :: + + '[{"product_id":"1520", "vendor_id":"8086"}]' + + :param whitelist_spec: A JSON string for a dictionary or list thereof. + Each dictionary specifies the pci device properties requirement. + See the definition of ``passthrough_whitelist`` in + ``nova.conf.pci`` for details and examples. + """ + if whitelist_spec: + self.specs = self._parse_white_list_from_config(whitelist_spec) + else: + self.specs = [] + + @staticmethod + def _parse_white_list_from_config(whitelists): + """Parse and validate the pci whitelist from the nova config.""" + specs = [] + for jsonspec in whitelists: + try: + dev_spec = jsonutils.loads(jsonspec) + except ValueError: + raise exception.PciConfigInvalidWhitelist( + reason=_("Invalid entry: '%s'") % jsonspec) + if isinstance(dev_spec, dict): + dev_spec = [dev_spec] + elif not isinstance(dev_spec, list): + raise exception.PciConfigInvalidWhitelist( + reason=_("Invalid entry: '%s'; " + "Expecting list or dict") % jsonspec) + + for ds in dev_spec: + if not isinstance(ds, dict): + raise exception.PciConfigInvalidWhitelist( + reason=_("Invalid entry: '%s'; " + "Expecting dict") % ds) + + spec = devspec.PciDeviceSpec(ds) + specs.append(spec) + + return specs + + def device_assignable(self, dev): + """Check if a device can be assigned to a guest. + + :param dev: A dictionary describing the device properties + """ + for spec in self.specs: + if spec.match(dev): + return True + return False + + def get_devspec(self, pci_dev): + for spec in self.specs: + if spec.match_pci_obj(pci_dev): + return spec diff --git a/cyborg/common/constants.py b/cyborg/common/constants.py index 845b09b2..ab1a9f42 100644 --- a/cyborg/common/constants.py +++ b/cyborg/common/constants.py @@ -88,6 +88,7 @@ RESOURCES = { "QAT": "CUSTOM_QAT", "NIC": "CUSTOM_NIC", "SSD": 'CUSTOM_SSD', + "PCI": 'CUSTOM_PCI', } diff --git a/cyborg/common/exception.py b/cyborg/common/exception.py index ba4e1ba7..6a3b2fd4 100644 --- a/cyborg/common/exception.py +++ b/cyborg/common/exception.py @@ -425,3 +425,12 @@ class FPGAProgramError(CyborgException): class PciDeviceNotFoundById(NotFound): _msg_fmt = _("PCI device %(id)s not found") + + +class PciConfigInvalidWhitelist(Invalid): + _msg_fmt = _("Invalid PCI devices whitelist config: %(reason)s.") + + +class PciDeviceInvalidDeviceName(CyborgException): + _msg_fmt = _("Invalid PCI whitelist: The PCI whitelist can specify " + "devname or address, but not both.") diff --git a/cyborg/conf/devices.py b/cyborg/conf/devices.py index 454f5b4a..2a70f79d 100644 --- a/cyborg/conf/devices.py +++ b/cyborg/conf/devices.py @@ -14,6 +14,15 @@ from oslo_config import cfg +pci_group = cfg.OptGroup( + name='pci', + title='PCI passthrough options') + +pci_opts = [ + cfg.MultiStrOpt('passthrough_whitelist', + default=[], + help=" ") +] nic_group = cfg.OptGroup( name='nic_devices', @@ -72,6 +81,8 @@ def register_opts(conf): conf.register_opts(nic_opts, group=nic_group) conf.register_group(gpu_group) conf.register_opts(vgpu_opts, group=gpu_group) + conf.register_group(pci_group) + conf.register_opts(pci_opts, group=pci_group) def register_dynamic_opts(conf): diff --git a/setup.cfg b/setup.cfg index 2c914ec1..38ded6df 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,6 +57,7 @@ cyborg.accelerator.driver = intel_qat_driver = cyborg.accelerator.drivers.qat.intel.driver:IntelQATDriver intel_nic_driver = cyborg.accelerator.drivers.nic.intel.driver:IntelNICDriver inspur_nvme_ssd_driver = cyborg.accelerator.drivers.ssd.inspur.driver:InspurNVMeSSDDriver + pci_driver = cyborg.accelerator.drivers.pci.pci.driver:PCIDriver oslo.config.opts = cyborg = cyborg.conf.opts:list_opts