pci unify management
bp pcie-unified-management Change-Id: I2a285b61c4f41e999670715dfa2cef8c910467b9 Change-Id: Iaddcd2d5dfef987b3d11d30a76e2775c8e60ba38
This commit is contained in:
songwenping
parent
63c239bf3e
commit
5f896d2931
0
cyborg/accelerator/drivers/pci/__init__.py
Normal file
0
cyborg/accelerator/drivers/pci/__init__.py
Normal file
29
cyborg/accelerator/drivers/pci/base.py
Normal file
29
cyborg/accelerator/drivers/pci/base.py
Normal file
@ -0,0 +1,29 @@
|
||||
# Copyright 2024 Inspur.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
"""
|
||||
Cyborg Pci driver implementation.
|
||||
"""
|
||||
|
||||
|
||||
class PciDriver(object):
|
||||
"""Base class for Pci drivers.
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def discover(self):
|
||||
raise NotImplementedError()
|
||||
290
cyborg/accelerator/drivers/pci/devspec.py
Normal file
290
cyborg/accelerator/drivers/pci/devspec.py
Normal file
@ -0,0 +1,290 @@
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import abc
|
||||
import re
|
||||
import string
|
||||
|
||||
import six
|
||||
|
||||
from cyborg.accelerator.drivers.pci import utils
|
||||
from cyborg.common import exception
|
||||
from cyborg.common.i18n import _
|
||||
|
||||
MAX_VENDOR_ID = 0xFFFF
|
||||
MAX_PRODUCT_ID = 0xFFFF
|
||||
MAX_FUNC = 0x7
|
||||
MAX_DOMAIN = 0xFFFF
|
||||
MAX_BUS = 0xFF
|
||||
MAX_SLOT = 0x1F
|
||||
ANY = '*'
|
||||
REGEX_ANY = '.*'
|
||||
|
||||
|
||||
@six.add_metaclass(abc.ABCMeta)
|
||||
class PciAddressSpec(object):
|
||||
"""Abstract class for all PCI address spec styles
|
||||
|
||||
This class checks the address fields of the pci.passthrough_whitelist
|
||||
"""
|
||||
|
||||
@abc.abstractmethod
|
||||
def match(self, pci_addr):
|
||||
pass
|
||||
|
||||
def is_single_address(self):
|
||||
return all([
|
||||
all(c in string.hexdigits for c in self.domain),
|
||||
all(c in string.hexdigits for c in self.bus),
|
||||
all(c in string.hexdigits for c in self.slot),
|
||||
all(c in string.hexdigits for c in self.func)])
|
||||
|
||||
def _set_pci_dev_info(self, prop, maxval, hex_value):
|
||||
a = getattr(self, prop)
|
||||
if a == ANY:
|
||||
return
|
||||
try:
|
||||
v = int(a, 16)
|
||||
except ValueError:
|
||||
raise exception.PciConfigInvalidWhitelist(
|
||||
reason=_("property %(property)s ('%(attr)s') does not parse "
|
||||
"as a hex number.") % {'property': prop, 'attr': a})
|
||||
if v > maxval:
|
||||
raise exception.PciConfigInvalidWhitelist(
|
||||
reason=_("property %(property)s (%(attr)s) is greater than "
|
||||
"the maximum allowable value (%(max)X).") % {
|
||||
'property': prop, 'attr': a, 'max': maxval})
|
||||
setattr(self, prop, hex_value % v)
|
||||
|
||||
|
||||
class PhysicalPciAddress(PciAddressSpec):
|
||||
"""Manages the address fields for a fully-qualified PCI address.
|
||||
|
||||
This function class will validate the address fields for a single
|
||||
PCI device.
|
||||
"""
|
||||
def __init__(self, pci_addr):
|
||||
try:
|
||||
if isinstance(pci_addr, dict):
|
||||
self.domain = pci_addr['domain']
|
||||
self.bus = pci_addr['bus']
|
||||
self.slot = pci_addr['slot']
|
||||
self.func = pci_addr['function']
|
||||
else:
|
||||
self.domain, self.bus, self.slot, self.func = (
|
||||
utils.get_pci_address_fields(pci_addr))
|
||||
self._set_pci_dev_info('func', MAX_FUNC, '%1x')
|
||||
self._set_pci_dev_info('domain', MAX_DOMAIN, '%04x')
|
||||
self._set_pci_dev_info('bus', MAX_BUS, '%02x')
|
||||
self._set_pci_dev_info('slot', MAX_SLOT, '%02x')
|
||||
except (KeyError, ValueError):
|
||||
raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
|
||||
|
||||
def match(self, phys_pci_addr):
|
||||
conditions = [
|
||||
self.domain == phys_pci_addr.domain,
|
||||
self.bus == phys_pci_addr.bus,
|
||||
self.slot == phys_pci_addr.slot,
|
||||
self.func == phys_pci_addr.func,
|
||||
]
|
||||
return all(conditions)
|
||||
|
||||
|
||||
class PciAddressGlobSpec(PciAddressSpec):
|
||||
"""Manages the address fields with glob style.
|
||||
|
||||
This function class will validate the address fields with glob style,
|
||||
check for wildcards, and insert wildcards where the field is left blank.
|
||||
"""
|
||||
|
||||
def __init__(self, pci_addr):
|
||||
self.domain = ANY
|
||||
self.bus = ANY
|
||||
self.slot = ANY
|
||||
self.func = ANY
|
||||
|
||||
dbs, sep, func = pci_addr.partition('.')
|
||||
if func:
|
||||
self.func = func.strip()
|
||||
self._set_pci_dev_info('func', MAX_FUNC, '%01x')
|
||||
if dbs:
|
||||
dbs_fields = dbs.split(':')
|
||||
if len(dbs_fields) > 3:
|
||||
raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
|
||||
# If we got a partial address like ":00.", we need to turn this
|
||||
# into a domain of ANY, a bus of ANY, and a slot of 00. This code
|
||||
# allows the address bus and/or domain to be left off
|
||||
dbs_all = [ANY] * (3 - len(dbs_fields))
|
||||
dbs_all.extend(dbs_fields)
|
||||
dbs_checked = [s.strip() or ANY for s in dbs_all]
|
||||
self.domain, self.bus, self.slot = dbs_checked
|
||||
self._set_pci_dev_info('domain', MAX_DOMAIN, '%04x')
|
||||
self._set_pci_dev_info('bus', MAX_BUS, '%02x')
|
||||
self._set_pci_dev_info('slot', MAX_SLOT, '%02x')
|
||||
|
||||
def match(self, phys_pci_addr):
|
||||
conditions = [
|
||||
self.domain in (ANY, phys_pci_addr.domain),
|
||||
self.bus in (ANY, phys_pci_addr.bus),
|
||||
self.slot in (ANY, phys_pci_addr.slot),
|
||||
self.func in (ANY, phys_pci_addr.func)
|
||||
]
|
||||
return all(conditions)
|
||||
|
||||
|
||||
class PciAddressRegexSpec(PciAddressSpec):
|
||||
"""Manages the address fields with regex style.
|
||||
|
||||
This function class will validate the address fields with regex style.
|
||||
The validation includes check for all PCI address attributes and validate
|
||||
their regex.
|
||||
"""
|
||||
def __init__(self, pci_addr):
|
||||
try:
|
||||
self.domain = pci_addr.get('domain', REGEX_ANY)
|
||||
self.bus = pci_addr.get('bus', REGEX_ANY)
|
||||
self.slot = pci_addr.get('slot', REGEX_ANY)
|
||||
self.func = pci_addr.get('function', REGEX_ANY)
|
||||
self.domain_regex = re.compile(self.domain)
|
||||
self.bus_regex = re.compile(self.bus)
|
||||
self.slot_regex = re.compile(self.slot)
|
||||
self.func_regex = re.compile(self.func)
|
||||
except re.error:
|
||||
raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
|
||||
|
||||
def match(self, phys_pci_addr):
|
||||
conditions = [
|
||||
bool(self.domain_regex.match(phys_pci_addr.domain)),
|
||||
bool(self.bus_regex.match(phys_pci_addr.bus)),
|
||||
bool(self.slot_regex.match(phys_pci_addr.slot)),
|
||||
bool(self.func_regex.match(phys_pci_addr.func))
|
||||
]
|
||||
return all(conditions)
|
||||
|
||||
|
||||
class WhitelistPciAddress(object):
|
||||
"""Manages the address fields of the whitelist.
|
||||
|
||||
This class checks the address fields of the pci.passthrough_whitelist
|
||||
configuration option, validating the address fields.
|
||||
Example configs:
|
||||
|
||||
| [pci]
|
||||
| passthrough_whitelist = {"address":"*:0a:00.*",
|
||||
| "physical_network":"physnet1"}
|
||||
| passthrough_whitelist = {"address": {"domain": ".*",
|
||||
"bus": "02",
|
||||
"slot": "01",
|
||||
"function": "[0-2]"},
|
||||
"physical_network":"net1"}
|
||||
| passthrough_whitelist = {"vendor_id":"1137","product_id":"0071"}
|
||||
|
||||
"""
|
||||
def __init__(self, pci_addr, is_physical_function):
|
||||
self.is_physical_function = is_physical_function
|
||||
self._init_address_fields(pci_addr)
|
||||
|
||||
def _check_physical_function(self):
|
||||
if self.pci_address_spec.is_single_address():
|
||||
self.is_physical_function = (
|
||||
utils.is_physical_function(
|
||||
self.pci_address_spec.domain,
|
||||
self.pci_address_spec.bus,
|
||||
self.pci_address_spec.slot,
|
||||
self.pci_address_spec.func))
|
||||
|
||||
def _init_address_fields(self, pci_addr):
|
||||
if not self.is_physical_function:
|
||||
if isinstance(pci_addr, six.string_types):
|
||||
self.pci_address_spec = PciAddressGlobSpec(pci_addr)
|
||||
elif isinstance(pci_addr, dict):
|
||||
self.pci_address_spec = PciAddressRegexSpec(pci_addr)
|
||||
else:
|
||||
raise exception.PciDeviceWrongAddressFormat(address=pci_addr)
|
||||
self._check_physical_function()
|
||||
else:
|
||||
self.pci_address_spec = PhysicalPciAddress(pci_addr)
|
||||
|
||||
def match(self, pci_addr, pci_phys_addr):
|
||||
"""Match a device to this PciAddress. Assume this is called given
|
||||
pci_addr and pci_phys_addr reported by libvirt, no attempt is made to
|
||||
verify if pci_addr is a VF of pci_phys_addr.
|
||||
|
||||
:param pci_addr: PCI address of the device to match.
|
||||
:param pci_phys_addr: PCI address of the parent of the device to match
|
||||
(or None if the device is not a VF).
|
||||
"""
|
||||
|
||||
# Try to match on the parent PCI address if the PciDeviceSpec is a
|
||||
# PF (sriov is available) and the device to match is a VF. This
|
||||
# makes it possible to specify the PCI address of a PF in the
|
||||
# pci.passthrough_whitelist to match any of its VFs' PCI addresses.
|
||||
if self.is_physical_function and pci_phys_addr:
|
||||
pci_phys_addr_obj = PhysicalPciAddress(pci_phys_addr)
|
||||
if self.pci_address_spec.match(pci_phys_addr_obj):
|
||||
return True
|
||||
|
||||
# Try to match on the device PCI address only.
|
||||
pci_addr_obj = PhysicalPciAddress(pci_addr)
|
||||
return self.pci_address_spec.match(pci_addr_obj)
|
||||
|
||||
|
||||
class PciDeviceSpec(PciAddressSpec):
|
||||
def __init__(self, dev_spec):
|
||||
self.tags = dev_spec
|
||||
self._init_dev_details()
|
||||
|
||||
def _init_dev_details(self):
|
||||
self.vendor_id = self.tags.pop("vendor_id", ANY)
|
||||
self.product_id = self.tags.pop("product_id", ANY)
|
||||
# Note(moshele): The address attribute can be a string or a dict.
|
||||
# For glob syntax or specific pci it is a string and for regex syntax
|
||||
# it is a dict. The WhitelistPciAddress class handles both types.
|
||||
self.address = self.tags.pop("address", None)
|
||||
self.dev_name = self.tags.pop("devname", None)
|
||||
|
||||
self.vendor_id = self.vendor_id.strip()
|
||||
self._set_pci_dev_info('vendor_id', MAX_VENDOR_ID, '%04x')
|
||||
self._set_pci_dev_info('product_id', MAX_PRODUCT_ID, '%04x')
|
||||
|
||||
if self.address and self.dev_name:
|
||||
raise exception.PciDeviceInvalidDeviceName()
|
||||
if not self.dev_name:
|
||||
pci_address = self.address or "*:*:*.*"
|
||||
self.address = WhitelistPciAddress(pci_address, False)
|
||||
|
||||
def match(self, dev_dict):
|
||||
if self.dev_name:
|
||||
address_str, pf = utils.get_function_by_ifname(
|
||||
self.dev_name)
|
||||
if not address_str:
|
||||
return False
|
||||
# Note(moshele): In this case we always passing a string
|
||||
# of the PF pci address
|
||||
address_obj = WhitelistPciAddress(address_str, pf)
|
||||
elif self.address:
|
||||
address_obj = self.address
|
||||
return all([
|
||||
self.vendor_id in (ANY, dev_dict['vendor_id']),
|
||||
self.product_id in (ANY, dev_dict['product_id']),
|
||||
address_obj.match(dev_dict['address'],
|
||||
dev_dict.get('parent_addr'))])
|
||||
|
||||
def match_pci_obj(self, pci_obj):
|
||||
return self.match({'vendor_id': pci_obj.vendor_id,
|
||||
'product_id': pci_obj.product_id,
|
||||
'address': pci_obj.address,
|
||||
'parent_addr': pci_obj.parent_addr})
|
||||
|
||||
def get_tags(self):
|
||||
return self.tags
|
||||
0
cyborg/accelerator/drivers/pci/pci/__init__.py
Normal file
0
cyborg/accelerator/drivers/pci/pci/__init__.py
Normal file
30
cyborg/accelerator/drivers/pci/pci/driver.py
Normal file
30
cyborg/accelerator/drivers/pci/pci/driver.py
Normal file
@ -0,0 +1,30 @@
|
||||
# Copyright 2024 Inspur.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
"""
|
||||
Cyborg PCI driver implementation.
|
||||
"""
|
||||
|
||||
from cyborg.accelerator.drivers.pci.base import PciDriver
|
||||
from cyborg.accelerator.drivers.pci.pci import sysinfo
|
||||
|
||||
|
||||
class PCIDriver(PciDriver):
|
||||
"""Class for Pci drivers.
|
||||
Vendor should implement their specific drivers in this class.
|
||||
"""
|
||||
|
||||
def discover(self):
|
||||
return sysinfo.discover()
|
||||
147
cyborg/accelerator/drivers/pci/pci/sysinfo.py
Normal file
147
cyborg/accelerator/drivers/pci/pci/sysinfo.py
Normal file
@ -0,0 +1,147 @@
|
||||
# Copyright 2024 Inspur.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
"""
|
||||
Cyborg PCI driver implementation.
|
||||
"""
|
||||
from oslo_log import log as logging
|
||||
from oslo_serialization import jsonutils
|
||||
|
||||
from cyborg.accelerator.common import utils
|
||||
from cyborg.accelerator.drivers.pci import utils as pci_utils
|
||||
from cyborg.accelerator.drivers.pci import whitelist
|
||||
from cyborg.common import constants
|
||||
import cyborg.conf
|
||||
from cyborg.objects.driver_objects import driver_attach_handle
|
||||
from cyborg.objects.driver_objects import driver_attribute
|
||||
from cyborg.objects.driver_objects import driver_controlpath_id
|
||||
from cyborg.objects.driver_objects import driver_deployable
|
||||
from cyborg.objects.driver_objects import driver_device
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = cyborg.conf.CONF
|
||||
|
||||
|
||||
def _get_traits(vendor_id, product_id):
|
||||
"""Generate traits for PCIs.
|
||||
: param vendor_id: vendor_id of PCI, eg."10de"
|
||||
: param product_id: product_id of PCI, eg."1eb8".
|
||||
|
||||
Example PGPU traits:
|
||||
{traits:["OWNER_CYBORG", "CUSTOM_PCI_1EB8"]}
|
||||
"""
|
||||
vendor_name = pci_utils.VENDOR_MAPS.get(vendor_id).upper()
|
||||
traits = ["CUSTOM_PCI_" + vendor_name]
|
||||
# PCIE trait
|
||||
product_trait = "_".join(('CUSTOM_PCI_PRODUCT_ID', product_id.upper()))
|
||||
traits.append(product_trait)
|
||||
return {"traits": traits}
|
||||
|
||||
|
||||
def _generate_attribute_list(pci):
|
||||
attr_list = []
|
||||
index = 0
|
||||
for k, v in pci.items():
|
||||
if k == "rc":
|
||||
driver_attr = driver_attribute.DriverAttribute()
|
||||
driver_attr.key, driver_attr.value = k, v
|
||||
attr_list.append(driver_attr)
|
||||
if k == "traits":
|
||||
values = pci.get(k, [])
|
||||
for val in values:
|
||||
driver_attr = driver_attribute.DriverAttribute(
|
||||
key="trait" + str(index), value=val)
|
||||
index = index + 1
|
||||
attr_list.append(driver_attr)
|
||||
return attr_list
|
||||
|
||||
|
||||
def _generate_attach_handle(pci):
|
||||
driver_ah = driver_attach_handle.DriverAttachHandle()
|
||||
driver_ah.in_use = False
|
||||
driver_ah.attach_type = constants.AH_TYPE_PCI
|
||||
driver_ah.attach_info = utils.pci_str_to_json(pci["devices"])
|
||||
return driver_ah
|
||||
|
||||
|
||||
def _generate_dep_list(pci):
|
||||
dep_list = []
|
||||
driver_dep = driver_deployable.DriverDeployable()
|
||||
driver_dep.attribute_list = _generate_attribute_list(pci)
|
||||
driver_dep.attach_handle_list = []
|
||||
# NOTE(wangzhh): The name of deployable should be unique, its format is
|
||||
# under disscussion, may looks like
|
||||
# <ComputeNodeName>_<NumaNodeName>_<CyborgName>_<NumInHost>
|
||||
# NOTE(yumeng) Since Wallaby release, the deplpyable_name is named as
|
||||
# <Compute_hostname>_<Device_address>
|
||||
driver_dep.name = pci.get('hostname', '') + '_' + pci["devices"]
|
||||
driver_dep.driver_name = \
|
||||
pci_utils.VENDOR_MAPS.get(pci["vendor_id"]).upper()
|
||||
driver_dep.num_accelerators = 1
|
||||
driver_dep.attach_handle_list = [_generate_attach_handle(pci)]
|
||||
dep_list.append(driver_dep)
|
||||
return dep_list, driver_dep.num_accelerators
|
||||
|
||||
|
||||
def _generate_controlpath_id(pci):
|
||||
driver_cpid = driver_controlpath_id.DriverControlPathID()
|
||||
driver_cpid.cpid_type = "PCI"
|
||||
driver_cpid.cpid_info = utils.pci_str_to_json(pci["devices"])
|
||||
return driver_cpid
|
||||
|
||||
|
||||
def _generate_driver_device(pci):
|
||||
driver_device_obj = driver_device.DriverDevice()
|
||||
driver_device_obj.vendor = pci['vendor_id']
|
||||
driver_device_obj.model = pci['product_id']
|
||||
std_board_info = {'product_id': pci.get('product_id'),
|
||||
'controller': pci.get('controller'),
|
||||
}
|
||||
driver_device_obj.std_board_info = jsonutils.dumps(std_board_info)
|
||||
driver_device_obj.type = constants.DEVICE_GPU
|
||||
driver_device_obj.stub = pci.get('stub', False)
|
||||
driver_device_obj.controlpath_id = _generate_controlpath_id(pci)
|
||||
driver_device_obj.deployable_list, ais = _generate_dep_list(pci)
|
||||
driver_device_obj.vendor_board_info = pci.get('vendor_board_info',
|
||||
"miss_vb_info")
|
||||
return driver_device_obj
|
||||
|
||||
|
||||
def _discover_pcis():
|
||||
cyborg.conf.devices.register_dynamic_opts(CONF)
|
||||
# discover pci devices by "lspci"
|
||||
pci_list = []
|
||||
pcis = pci_utils.get_pci_devices()
|
||||
LOG.info('pcis:%s', pcis)
|
||||
# report trait,rc and generate driver object
|
||||
dev_filter = whitelist.Whitelist(CONF.pci.passthrough_whitelist)
|
||||
for pci in pcis:
|
||||
m = dev_filter.device_assignable(pci)
|
||||
if m:
|
||||
pci_dict = m.groupdict()
|
||||
# get hostname for deployable_name usage
|
||||
pci_dict['hostname'] = CONF.host
|
||||
pci_dict["rc"] = constants.RESOURCES["PCI"]
|
||||
traits = _get_traits(pci_dict["vendor_id"],
|
||||
pci_dict["product_id"])
|
||||
pci_dict.update(traits)
|
||||
pci_list.append(_generate_driver_device(pci_dict))
|
||||
LOG.info('pci_list:%s', pci_list)
|
||||
return pci_list
|
||||
|
||||
|
||||
def discover():
|
||||
devs = _discover_pcis()
|
||||
return devs
|
||||
232
cyborg/accelerator/drivers/pci/utils.py
Normal file
232
cyborg/accelerator/drivers/pci/utils.py
Normal file
@ -0,0 +1,232 @@
|
||||
# Copyright 2024 Inspur.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
|
||||
from oslo_concurrency import processutils
|
||||
from oslo_log import log as logging
|
||||
import six
|
||||
|
||||
from cyborg.common import exception
|
||||
import cyborg.privsep
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
PCI_VENDOR_PATTERN = "^(hex{4})$".replace("hex", r"[\da-fA-F]")
|
||||
_PCI_ADDRESS_PATTERN = ("^(hex{4}):(hex{2}):(hex{2}).(oct{1})$".
|
||||
replace("hex", r"[\da-fA-F]").
|
||||
replace("oct", "[0-7]"))
|
||||
_PCI_ADDRESS_REGEX = re.compile(_PCI_ADDRESS_PATTERN)
|
||||
|
||||
_SRIOV_TOTALVFS = "sriov_totalvfs"
|
||||
|
||||
VENDOR_MAPS = {"10de": "nvidia", "102b": "matrox"}
|
||||
|
||||
|
||||
@cyborg.privsep.sys_admin_pctxt.entrypoint
|
||||
def get_pci_devices():
|
||||
cmd = ['lspci', '-nnn', '-D']
|
||||
return processutils.execute(*cmd)
|
||||
|
||||
|
||||
def pci_device_prop_match(pci_dev, specs):
|
||||
"""Check if the pci_dev meet spec requirement
|
||||
|
||||
Specs is a list of PCI device property requirements.
|
||||
An example of device requirement that the PCI should be either:
|
||||
a) Device with vendor_id as 0x8086 and product_id as 0x8259, or
|
||||
b) Device with vendor_id as 0x10de and product_id as 0x10d8:
|
||||
|
||||
[{"vendor_id":"8086", "product_id":"8259"},
|
||||
{"vendor_id":"10de", "product_id":"10d8",
|
||||
"capabilities_network": ["rx", "tx", "tso", "gso"]}]
|
||||
|
||||
"""
|
||||
def _matching_devices(spec):
|
||||
for k, v in spec.items():
|
||||
pci_dev_v = pci_dev.get(k)
|
||||
if isinstance(v, list) and isinstance(pci_dev_v, list):
|
||||
if not all(x in pci_dev.get(k) for x in v):
|
||||
return False
|
||||
else:
|
||||
# We don't need to check case for tags in order to avoid any
|
||||
# mismatch with the tags provided by users for port
|
||||
# binding profile and the ones configured by operators
|
||||
# with pci whitelist option.
|
||||
if isinstance(v, six.string_types):
|
||||
v = v.lower()
|
||||
if isinstance(pci_dev_v, six.string_types):
|
||||
pci_dev_v = pci_dev_v.lower()
|
||||
if pci_dev_v != v:
|
||||
return False
|
||||
return True
|
||||
|
||||
return any(_matching_devices(spec) for spec in specs)
|
||||
|
||||
|
||||
def parse_address(address):
|
||||
"""Returns (domain, bus, slot, function) from PCI address that is stored in
|
||||
PciDevice DB table.
|
||||
"""
|
||||
m = _PCI_ADDRESS_REGEX.match(address)
|
||||
if not m:
|
||||
raise exception.PciDeviceWrongAddressFormat(address=address)
|
||||
return m.groups()
|
||||
|
||||
|
||||
def get_pci_address_fields(pci_addr):
|
||||
"""Parse a fully-specified PCI device address.
|
||||
|
||||
Does not validate that the components are valid hex or wildcard values.
|
||||
|
||||
:param pci_addr: A string of the form "<domain>:<bus>:<slot>.<function>".
|
||||
:return: A 4-tuple of strings ("<domain>", "<bus>", "<slot>", "<function>")
|
||||
"""
|
||||
dbs, sep, func = pci_addr.partition('.')
|
||||
domain, bus, slot = dbs.split(':')
|
||||
return domain, bus, slot, func
|
||||
|
||||
|
||||
def get_pci_address(domain, bus, slot, func):
|
||||
"""Assembles PCI address components into a fully-specified PCI address.
|
||||
|
||||
Does not validate that the components are valid hex or wildcard values.
|
||||
|
||||
:param domain, bus, slot, func: Hex or wildcard strings.
|
||||
:return: A string of the form "<domain>:<bus>:<slot>.<function>".
|
||||
"""
|
||||
return '%s:%s:%s.%s' % (domain, bus, slot, func)
|
||||
|
||||
|
||||
def get_function_by_ifname(ifname):
|
||||
"""Given the device name, returns the PCI address of a device
|
||||
and returns True if the address is in a physical function.
|
||||
"""
|
||||
dev_path = "/sys/class/net/%s/device" % ifname
|
||||
if os.path.isdir(dev_path):
|
||||
try:
|
||||
# sriov_totalvfs contains the maximum possible VFs for this PF
|
||||
with open(os.path.join(dev_path, _SRIOV_TOTALVFS)) as fd:
|
||||
sriov_totalvfs = int(fd.read())
|
||||
return (os.readlink(dev_path).strip("./"),
|
||||
sriov_totalvfs > 0)
|
||||
except (IOError, ValueError):
|
||||
return os.readlink(dev_path).strip("./"), False
|
||||
return None, False
|
||||
|
||||
|
||||
def is_physical_function(domain, bus, slot, function):
|
||||
dev_path = "/sys/bus/pci/devices/%(d)s:%(b)s:%(s)s.%(f)s/" % {
|
||||
"d": domain, "b": bus, "s": slot, "f": function}
|
||||
if os.path.isdir(dev_path):
|
||||
try:
|
||||
with open(dev_path + _SRIOV_TOTALVFS) as fd:
|
||||
sriov_totalvfs = int(fd.read())
|
||||
return sriov_totalvfs > 0
|
||||
except (IOError, ValueError):
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _get_sysfs_netdev_path(pci_addr, pf_interface):
|
||||
"""Get the sysfs path based on the PCI address of the device.
|
||||
|
||||
Assumes a networking device - will not check for the existence of the path.
|
||||
"""
|
||||
if pf_interface:
|
||||
return "/sys/bus/pci/devices/%s/physfn/net" % pci_addr
|
||||
return "/sys/bus/pci/devices/%s/net" % pci_addr
|
||||
|
||||
|
||||
def get_ifname_by_pci_address(pci_addr, pf_interface=False):
|
||||
"""Get the interface name based on a VF's pci address.
|
||||
|
||||
The returned interface name is either the parent PF's or that of the VF
|
||||
itself based on the argument of pf_interface.
|
||||
"""
|
||||
dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface)
|
||||
try:
|
||||
dev_info = os.listdir(dev_path)
|
||||
return dev_info.pop()
|
||||
except Exception:
|
||||
raise exception.PciDeviceNotFoundById(id=pci_addr)
|
||||
|
||||
|
||||
def get_mac_by_pci_address(pci_addr, pf_interface=False):
|
||||
"""Get the MAC address of the nic based on its PCI address.
|
||||
|
||||
Raises PciDeviceNotFoundById in case the pci device is not a NIC
|
||||
"""
|
||||
dev_path = _get_sysfs_netdev_path(pci_addr, pf_interface)
|
||||
if_name = get_ifname_by_pci_address(pci_addr, pf_interface)
|
||||
addr_file = os.path.join(dev_path, if_name, 'address')
|
||||
|
||||
try:
|
||||
with open(addr_file) as f:
|
||||
mac = next(f).strip()
|
||||
return mac
|
||||
except (IOError, StopIteration) as e:
|
||||
LOG.warning("Could not find the expected sysfs file for "
|
||||
"determining the MAC address of the PCI device "
|
||||
"%(addr)s. May not be a NIC. Error: %(e)s",
|
||||
{'addr': pci_addr, 'e': e})
|
||||
raise exception.PciDeviceNotFoundById(id=pci_addr)
|
||||
|
||||
|
||||
def get_vf_num_by_pci_address(pci_addr):
|
||||
"""Get the VF number based on a VF's pci address
|
||||
|
||||
A VF is associated with an VF number, which ip link command uses to
|
||||
configure it. This number can be obtained from the PCI device filesystem.
|
||||
"""
|
||||
VIRTFN_RE = re.compile(r"virtfn(\d+)")
|
||||
virtfns_path = "/sys/bus/pci/devices/%s/physfn/virtfn*" % (pci_addr)
|
||||
vf_num = None
|
||||
try:
|
||||
for vf_path in glob.iglob(virtfns_path):
|
||||
if re.search(pci_addr, os.readlink(vf_path)):
|
||||
t = VIRTFN_RE.search(vf_path)
|
||||
vf_num = t.group(1)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
if vf_num is None:
|
||||
raise exception.PciDeviceNotFoundById(id=pci_addr)
|
||||
return vf_num
|
||||
|
||||
|
||||
def get_net_name_by_vf_pci_address(vfaddress):
|
||||
"""Given the VF PCI address, returns the net device name.
|
||||
|
||||
Every VF is associated to a PCI network device. This function
|
||||
returns the libvirt name given to this network device; e.g.:
|
||||
|
||||
<device>
|
||||
<name>net_enp8s0f0_90_e2_ba_5e_a6_40</name>
|
||||
...
|
||||
|
||||
In the libvirt parser information tree, the network device stores the
|
||||
network capabilities associated to this device.
|
||||
"""
|
||||
try:
|
||||
mac = get_mac_by_pci_address(vfaddress).split(':')
|
||||
ifname = get_ifname_by_pci_address(vfaddress)
|
||||
return ("net_%(ifname)s_%(mac)s" %
|
||||
{'ifname': ifname, 'mac': '_'.join(mac)})
|
||||
except Exception:
|
||||
LOG.warning("No net device was found for VF %(vfaddress)s",
|
||||
{'vfaddress': vfaddress})
|
||||
return
|
||||
91
cyborg/accelerator/drivers/pci/whitelist.py
Normal file
91
cyborg/accelerator/drivers/pci/whitelist.py
Normal file
@ -0,0 +1,91 @@
|
||||
# Copyright 2024 Inspur.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_serialization import jsonutils
|
||||
|
||||
from cyborg.accelerator.drivers.pci import devspec
|
||||
from cyborg.common import exception
|
||||
from cyborg.common.i18n import _
|
||||
|
||||
|
||||
class Whitelist(object):
|
||||
"""White list class to represent assignable pci devices.
|
||||
|
||||
Not all devices on a compute node can be assigned to a guest. The cloud
|
||||
administrator decides which devices can be assigned based on ``vendor_id``
|
||||
or ``product_id``, etc. If no white list is specified, no devices will be
|
||||
assignable.
|
||||
"""
|
||||
|
||||
def __init__(self, whitelist_spec=None):
|
||||
"""White list constructor
|
||||
|
||||
For example, the following json string specifies that devices whose
|
||||
vendor_id is '8086' and product_id is '1520' can be assigned
|
||||
to guests. ::
|
||||
|
||||
'[{"product_id":"1520", "vendor_id":"8086"}]'
|
||||
|
||||
:param whitelist_spec: A JSON string for a dictionary or list thereof.
|
||||
Each dictionary specifies the pci device properties requirement.
|
||||
See the definition of ``passthrough_whitelist`` in
|
||||
``nova.conf.pci`` for details and examples.
|
||||
"""
|
||||
if whitelist_spec:
|
||||
self.specs = self._parse_white_list_from_config(whitelist_spec)
|
||||
else:
|
||||
self.specs = []
|
||||
|
||||
@staticmethod
|
||||
def _parse_white_list_from_config(whitelists):
|
||||
"""Parse and validate the pci whitelist from the nova config."""
|
||||
specs = []
|
||||
for jsonspec in whitelists:
|
||||
try:
|
||||
dev_spec = jsonutils.loads(jsonspec)
|
||||
except ValueError:
|
||||
raise exception.PciConfigInvalidWhitelist(
|
||||
reason=_("Invalid entry: '%s'") % jsonspec)
|
||||
if isinstance(dev_spec, dict):
|
||||
dev_spec = [dev_spec]
|
||||
elif not isinstance(dev_spec, list):
|
||||
raise exception.PciConfigInvalidWhitelist(
|
||||
reason=_("Invalid entry: '%s'; "
|
||||
"Expecting list or dict") % jsonspec)
|
||||
|
||||
for ds in dev_spec:
|
||||
if not isinstance(ds, dict):
|
||||
raise exception.PciConfigInvalidWhitelist(
|
||||
reason=_("Invalid entry: '%s'; "
|
||||
"Expecting dict") % ds)
|
||||
|
||||
spec = devspec.PciDeviceSpec(ds)
|
||||
specs.append(spec)
|
||||
|
||||
return specs
|
||||
|
||||
def device_assignable(self, dev):
|
||||
"""Check if a device can be assigned to a guest.
|
||||
|
||||
:param dev: A dictionary describing the device properties
|
||||
"""
|
||||
for spec in self.specs:
|
||||
if spec.match(dev):
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_devspec(self, pci_dev):
|
||||
for spec in self.specs:
|
||||
if spec.match_pci_obj(pci_dev):
|
||||
return spec
|
||||
@ -88,6 +88,7 @@ RESOURCES = {
|
||||
"QAT": "CUSTOM_QAT",
|
||||
"NIC": "CUSTOM_NIC",
|
||||
"SSD": 'CUSTOM_SSD',
|
||||
"PCI": 'CUSTOM_PCI',
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -425,3 +425,12 @@ class FPGAProgramError(CyborgException):
|
||||
|
||||
class PciDeviceNotFoundById(NotFound):
|
||||
_msg_fmt = _("PCI device %(id)s not found")
|
||||
|
||||
|
||||
class PciConfigInvalidWhitelist(Invalid):
|
||||
_msg_fmt = _("Invalid PCI devices whitelist config: %(reason)s.")
|
||||
|
||||
|
||||
class PciDeviceInvalidDeviceName(CyborgException):
|
||||
_msg_fmt = _("Invalid PCI whitelist: The PCI whitelist can specify "
|
||||
"devname or address, but not both.")
|
||||
|
||||
@ -14,6 +14,15 @@
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
pci_group = cfg.OptGroup(
|
||||
name='pci',
|
||||
title='PCI passthrough options')
|
||||
|
||||
pci_opts = [
|
||||
cfg.MultiStrOpt('passthrough_whitelist',
|
||||
default=[],
|
||||
help=" ")
|
||||
]
|
||||
|
||||
nic_group = cfg.OptGroup(
|
||||
name='nic_devices',
|
||||
@ -72,6 +81,8 @@ def register_opts(conf):
|
||||
conf.register_opts(nic_opts, group=nic_group)
|
||||
conf.register_group(gpu_group)
|
||||
conf.register_opts(vgpu_opts, group=gpu_group)
|
||||
conf.register_group(pci_group)
|
||||
conf.register_opts(pci_opts, group=pci_group)
|
||||
|
||||
|
||||
def register_dynamic_opts(conf):
|
||||
|
||||
@ -57,6 +57,7 @@ cyborg.accelerator.driver =
|
||||
intel_qat_driver = cyborg.accelerator.drivers.qat.intel.driver:IntelQATDriver
|
||||
intel_nic_driver = cyborg.accelerator.drivers.nic.intel.driver:IntelNICDriver
|
||||
inspur_nvme_ssd_driver = cyborg.accelerator.drivers.ssd.inspur.driver:InspurNVMeSSDDriver
|
||||
pci_driver = cyborg.accelerator.drivers.pci.pci.driver:PCIDriver
|
||||
|
||||
oslo.config.opts =
|
||||
cyborg = cyborg.conf.opts:list_opts
|
||||
|
||||
Loading…
Reference in New Issue
Block a user