Add xilinx fpga driver

This patch implemented Xilinx FPGA driver in cyborg. Currently can
 bind xilinx card to guest, and support programming.

Implements: blueprint add-xilinx-fpga-driver
Change-Id: I8cf48f64a0754ff13da2dddc4fabc601087b14b1
This commit is contained in:
ericxiett 2021-08-11 16:55:56 +08:00
parent 828fed561c
commit 2c2ab34ee3
12 changed files with 497 additions and 4 deletions

View File

@ -0,0 +1,74 @@
# Copyright 2021 Inspur, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg Xilinx FPGA driver implementation.
"""
from oslo_concurrency import processutils
from cyborg.accelerator.drivers.fpga.base import FPGADriver
from cyborg.accelerator.drivers.fpga.xilinx import sysinfo
from cyborg.common import exception
import cyborg.privsep
@cyborg.privsep.sys_admin_pctxt.entrypoint
def _fpga_program_privileged(cmd_args):
# Example: xbmgmt program --device 0000:d8:00.0
# --base --image xilinx-u250-gen3x16-base
cmd = ["/opt/xilinx/xrt/bin/xbmgmt"]
cmd.extend(cmd_args)
return processutils.execute(*cmd)
class XilinxFPGADriver(FPGADriver):
"""Class for Xilinx FPGA drivers.
Vendor should implement their specific drivers in this class.
"""
VENDOR = "xilinx"
def __init__(self, *args, **kwargs):
pass
def discover(self):
return sysinfo.fpga_tree()
def program(self, controlpath_id, image_file_path):
"""Program the FPGA with the provided bitstream image.
:param controlpath_id: the ID of controlpath.
:param image_file_path: the path of the image file
:returns: True on success, False on failure
"""
if controlpath_id['cpid_type'] != "PCI":
raise exception.InvalidType(obj='controlpath_id',
type=controlpath_id['cpid_type'],
expected='PCI')
cmd_args = ['program']
cmd_args.append('--device')
bdf_dict = controlpath_id['cpid_info']
# BDF format: domain:bus:device:function
bdf = ':'.join([s for s in map(lambda x: bdf_dict[x],
['domain', 'bus', 'device', 'function'])])
cmd_args.append(bdf)
cmd_args.append('--base')
cmd_args.append('--image')
cmd_args.append(image_file_path)
try:
_fpga_program_privileged(cmd_args)
return True
except Exception:
return False

View File

@ -0,0 +1,188 @@
# Copyright 2021 Inspur, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg Xilinx FPGA driver implementation.
"""
import re
from oslo_concurrency import processutils
from oslo_log import log as logging
from oslo_serialization import jsonutils
from cyborg.accelerator.common import utils
from cyborg.common import constants
from cyborg.conf import CONF
from cyborg.objects.driver_objects import driver_attach_handle
from cyborg.objects.driver_objects import driver_attribute
from cyborg.objects.driver_objects import driver_controlpath_id
from cyborg.objects.driver_objects import driver_deployable
from cyborg.objects.driver_objects import driver_device
from cyborg.privsep import sys_admin_pctxt
LOG = logging.getLogger(__name__)
XILINX_FPGA_FLAGS = ["Xilinx Corporation Device",
"Processing accelerators"]
XILINX_FPGA_INFO_PATTERN = re.compile(
r"(?P<pci_addr>[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:"
r"[0-9a-fA-F]{2}\.[0-9a-fA-F]) "
r"(?P<controller>.*) [\[].*]: (?P<model>.*) .*"
r"[\[](?P<vendor_id>[0-9a-fA-F]"
r"{4}):(?P<product_id>[0-9a-fA-F]{4})].*")
XILINX_PF_MAPS = {"mgmt": "xclmgmt", "user": "xocl"}
VENDOR_MAPS = {"10ee": "xilinx"}
@sys_admin_pctxt.entrypoint
def lspci_privileged(cmd):
return processutils.execute(*cmd)
def get_pci_devices(pci_flags, vendor_id=None):
device_for_vendor_out = []
all_device_out = []
cmd = ['lspci', '-nnn', '-D']
lspci_out = lspci_privileged(cmd)[0].split('\n')
for i in range(len(lspci_out)):
if all(x in lspci_out[i] for x in pci_flags):
all_device_out.append(lspci_out[i])
if vendor_id and vendor_id in lspci_out[i]:
device_for_vendor_out.append(lspci_out[i])
return device_for_vendor_out if vendor_id else all_device_out
def _generate_traits(vendor_id, product_id):
"""Generate traits for FPGAs.
: param vendor_id: vendor_id of FPGA
: param product_id: product_id of FPGA
Example FPGA traits:
{traits:["CUSTOM_FPGA_XILINX", "CUSTOM_FPGA_PRODUCT_ID_5001"]}
"""
traits = []
traits.append("CUSTOM_FPGA_" + VENDOR_MAPS.get(vendor_id, "").upper())
traits.append("CUSTOM_FPGA_PRODUCT_ID_" + product_id.upper())
return {"traits": traits}
def _get_pf_type(device):
cmd = ['lspci', '-k', '-s', device]
result = lspci_privileged(cmd)[0]
for k, v in XILINX_PF_MAPS.items():
if v in result:
return k
def _combine_device_by_pci_func(pci_devices):
fpga_devices = []
for pci_dev in pci_devices:
m = XILINX_FPGA_INFO_PATTERN.match(pci_dev)
if m:
pci_dict = m.groupdict()
LOG.debug('Xilinx fpga pci device in dict: %s', pci_dict)
# 0000:3b:00.0/1
is_existed = False
new_addr = pci_dict.get('pci_addr')
for fpga in fpga_devices:
existed_addr = fpga.get('pci_addr')[0]
# compare domain:bus:slot
if existed_addr and \
new_addr.split('.')[0] == existed_addr.split('.')[0]:
fpga.update({'pci_addr': [existed_addr, new_addr]})
is_existed = True
if not is_existed:
traits = _generate_traits(pci_dict["vendor_id"],
pci_dict["product_id"])
pci_dict["rc"] = constants.RESOURCES["FPGA"]
pci_dict.update(traits)
pci_dict.update({'pci_addr': [new_addr]})
fpga_devices.append(pci_dict)
return fpga_devices
def _generate_controlpath_id(fpga):
driver_cpid = driver_controlpath_id.DriverControlPathID()
driver_cpid.cpid_type = "PCI"
driver_cpid.cpid_info = utils.pci_str_to_json(fpga['pci_addr'][0])
return driver_cpid
def _generate_dep_list(fpga):
dep_list = []
driver_dep = driver_deployable.DriverDeployable()
driver_dep.attribute_list = _generate_attribute_list(fpga)
driver_dep.attach_handle_list = []
driver_dep.name = CONF.host + '_' + fpga["pci_addr"][0]
driver_dep.driver_name = VENDOR_MAPS.get(fpga["vendor_id"]).upper()
driver_dep.num_accelerators = 1
driver_dep.attach_handle_list = _generate_attach_handle(fpga)
dep_list.append(driver_dep)
return dep_list
def _generate_attribute_list(fpga):
attr_list = []
for k, v in fpga.items():
if k == "rc":
driver_attr = driver_attribute.DriverAttribute()
driver_attr.key, driver_attr.value = k, v
attr_list.append(driver_attr)
if k == "traits":
values = fpga.get(k, [])
for index, val in enumerate(values):
driver_attr = driver_attribute.DriverAttribute(
key="trait" + str(index), value=val)
attr_list.append(driver_attr)
return attr_list
def _generate_attach_handle(fpga):
driver_ahs = []
for addr in fpga.get('pci_addr'):
driver_ah = driver_attach_handle.DriverAttachHandle()
driver_ah.attach_type = constants.AH_TYPE_PCI
driver_ah.attach_info = utils.pci_str_to_json(addr)
driver_ah.in_use = False
driver_ahs.append(driver_ah)
return driver_ahs
def fpga_tree():
fpga_list = []
fpga_pci_devices = get_pci_devices(XILINX_FPGA_FLAGS)
LOG.debug("Xilinx fpga devices from lspci: %s", fpga_pci_devices)
# In the return pci devices, mgmt pf and user pf are two entries.
# Now only when binding both to vm, end user can program it.
# So combine these two entries into one device.
for fpga in _combine_device_by_pci_func(fpga_pci_devices):
driver_device_obj = driver_device.DriverDevice()
driver_device_obj.vendor = fpga["vendor_id"]
driver_device_obj.model = fpga.get('model', 'miss model info')
std_board_info = {'product_id': fpga.get('product_id', None),
'controller': fpga.get('controller', None)}
vendor_board_info = {
'vendor_info': fpga.get('vendor_info', 'fpga_vb_info')}
driver_device_obj.std_board_info = jsonutils.dumps(std_board_info)
driver_device_obj.vendor_board_info = \
jsonutils.dumps(vendor_board_info)
driver_device_obj.type = constants.DEVICE_FPGA
driver_device_obj.stub = fpga.get('stub', False)
driver_device_obj.controlpath_id = _generate_controlpath_id(fpga)
driver_device_obj.deployable_list = _generate_dep_list(fpga)
fpga_list.append(driver_device_obj)
return fpga_list

View File

@ -135,9 +135,18 @@ class ARQsController(base.CyborgController):
extarq_list = []
for group_id, group in enumerate(devprof.groups):
accel_resources = [
int(val) for key, val in group.items()
if key.startswith('resources')]
accel_resources = []
# If the device profile requires the Xilinx fpga, the number of
# resources should multiply by 2 cause that end user can program
# the device only when both MGMT and USER PF are bound to
# instance.
if group.get("trait:CUSTOM_FPGA_XILINX") == "required":
accel_resources = [int(group.get("resources:FPGA"))] * 2
else:
accel_resources = [
int(val) for key, val in group.items()
if key.startswith('resources')]
# If/when we introduce non-accelerator resources, like
# device-local memory, the key search above needs to be
# made specific to accelerator resources only.

View File

@ -15,6 +15,7 @@
from cyborg.accelerator.drivers.fpga.base import FPGADriver
from cyborg.accelerator.drivers.fpga.intel.driver import IntelFPGADriver # noqa
from cyborg.accelerator.drivers.fpga.inspur.driver import InspurFPGADriver # noqa
from cyborg.accelerator.drivers.fpga.xilinx.driver import XilinxFPGADriver # noqa
from cyborg.tests import base
@ -22,7 +23,8 @@ class TestFPGADriver(base.TestCase):
def test_create(self):
FPGADriver.create("intel")
FPGADriver.create("inspur")
self.assertRaises(LookupError, FPGADriver.create, "xilinx")
FPGADriver.create("xilinx")
self.assertRaises(LookupError, FPGADriver.create, "fake")
def test_discover(self):
d = FPGADriver()

View File

@ -0,0 +1,129 @@
# Copyright 2021 Inspur, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_serialization import jsonutils
from unittest import mock
from cyborg.accelerator.drivers.fpga.xilinx.driver import XilinxFPGADriver
from cyborg.tests import base
XILINX_FPGA_INFO = ["0000:3b:00.0 Processing accelerators [1200]: "
"Xilinx Corporation Device [10ee:5000]\n"
"0000:3b:00.1 Processing accelerators [1200]: "
"Xilinx Corporation Device [10ee:5001]"]
def fake_output(arg):
if arg == ['lspci', '-nnn', '-D']:
return XILINX_FPGA_INFO
class TestXilinxFPGADriver(base.TestCase):
def setUp(self):
super(TestXilinxFPGADriver, self).setUp()
@mock.patch('cyborg.accelerator.drivers.fpga.'
'xilinx.sysinfo.lspci_privileged')
def test_discover(self, mock_devices_for_vendor):
mock_devices_for_vendor.side_effect = fake_output
self.set_defaults(host='fake-host', debug=True)
fpga_list = XilinxFPGADriver().discover()
self.assertEqual(1, len(fpga_list))
attach_handle_list = [
{'attach_type': 'PCI',
'attach_info': '{"bus": "3b", '
'"device": "00", '
'"domain": "0000", '
'"function": "0"}',
'in_use': False},
{'attach_type': 'PCI',
'attach_info': '{"bus": "3b", '
'"device": "00", '
'"domain": "0000", '
'"function": "1"}',
'in_use': False}
]
attribute_list = [
{'key': 'rc', 'value': 'FPGA'},
{'key': 'trait0', 'value': 'CUSTOM_FPGA_XILINX'},
{'key': 'trait1', 'value': 'CUSTOM_FPGA_PRODUCT_ID_5000'},
]
expected = {
'vendor': '10ee',
'type': 'FPGA',
'std_board_info': {"controller": "Processing accelerators",
"product_id": "5000"},
'vendor_board_info': {"vendor_info": "fpga_vb_info"},
'deployable_list':
[
{
'num_accelerators': 1,
'driver_name': 'XILINX',
'name': 'fake-host_0000:3b:00.0',
'attach_handle_list': attach_handle_list,
'attribute_list': attribute_list
},
],
'controlpath_id': {'cpid_info': '{"bus": "3b", '
'"device": "00", '
'"domain": "0000", '
'"function": "0"}',
'cpid_type': 'PCI'}
}
fpga_obj = fpga_list[0]
fpga_dict = fpga_obj.as_dict()
fpga_dep_list = fpga_dict['deployable_list']
fpga_attach_handle_list = (
fpga_dep_list[0].as_dict()['attach_handle_list'])
fpga_attribute_list = fpga_dep_list[0].as_dict()['attribute_list']
attri_obj_data = []
[attri_obj_data.append(attr.as_dict()) for attr in fpga_attribute_list]
attribute_actual_data = sorted(attri_obj_data, key=lambda i: i['key'])
self.assertEqual(expected['vendor'], fpga_dict['vendor'])
self.assertEqual(expected['controlpath_id'],
fpga_dict['controlpath_id'])
self.assertEqual(expected['std_board_info'],
jsonutils.loads(fpga_dict['std_board_info']))
self.assertEqual(expected['vendor_board_info'],
jsonutils.loads(fpga_dict['vendor_board_info']))
self.assertEqual(expected['deployable_list'][0]['num_accelerators'],
fpga_dep_list[0].as_dict()['num_accelerators'])
self.assertEqual(expected['deployable_list'][0]['name'],
fpga_dep_list[0].as_dict()['name'])
self.assertEqual(expected['deployable_list'][0]['driver_name'],
fpga_dep_list[0].as_dict()['driver_name'])
self.assertEqual(2, len(fpga_attach_handle_list))
self.assertEqual(attach_handle_list[0],
fpga_attach_handle_list[0].as_dict())
self.assertEqual(attach_handle_list[1],
fpga_attach_handle_list[1].as_dict())
self.assertEqual(attribute_list, attribute_actual_data)
@mock.patch('cyborg.accelerator.drivers.fpga.xilinx.driver.'
'_fpga_program_privileged')
def test_program(self, mock_prog):
bdf = '0000:3b:00:0'
expect_cmd_args = ['program', '--device', bdf, '--base',
'--image', '/path/image']
xilinx_driver = XilinxFPGADriver()
cpid_info = {"domain": "0000", "bus": "3b",
"device": "00", "function": "0"}
cpid = {'cpid_type': 'PCI', 'cpid_info': cpid_info}
# program PF
mock_prog.return_value = bytes([0])
xilinx_driver.program(cpid, "/path/image")
mock_prog.assert_called_with(expect_cmd_args)

View File

@ -203,6 +203,26 @@ class TestARQsController(v2_test.APITestV2):
dp_group_id = idx
self.assertEqual(dp_group_id, out_arq['device_profile_group_id'])
@mock.patch('cyborg.objects.DeviceProfile.get_by_name')
@mock.patch('cyborg.conductor.rpcapi.ConductorAPI.arq_create')
def test_create_with_xilinx_fpga(self, mock_obj_extarq, mock_obj_dp):
xilinx_fpga_dp = fake_device_profile.get_xilinx_fpga_devprof()
mock_obj_dp.return_value = dp = xilinx_fpga_dp
fake_xilinx_fpga_objs = fake_extarq.get_fake_xilinx_fpga_extarq_objs()
mock_obj_extarq.side_effect = fake_xilinx_fpga_objs
params = {'device_profile_name': dp['name']}
response = self.post_json(self.ARQ_URL, params, headers=self.headers)
data = jsonutils.loads(response.__dict__['controller_output'])
out_arqs = data['arqs']
self.assertEqual(HTTPStatus.CREATED, response.status_int)
self.assertEqual(len(out_arqs), 2)
for in_extarq, out_arq in zip(fake_xilinx_fpga_objs, out_arqs):
self._validate_arq(in_extarq.arq, out_arq)
for idx, out_arq in enumerate(out_arqs):
dp_group_id = idx
self.assertEqual(dp_group_id, out_arq['device_profile_group_id'])
@mock.patch('cyborg.objects.DeviceProfile.get_by_name')
@mock.patch('cyborg.objects.ExtARQ.create')
def test_create_with_wrong_dp(self, mock_obj_extarq, mock_obj_dp):

View File

@ -106,3 +106,23 @@ def get_db_devprofs():
dp_list = _get_device_profiles_as_dict()
db_devprofs = list(map(_convert_to_db_devprof, dp_list))
return db_devprofs
def get_xilinx_fpga_devprof():
xilinx_fpga_dp = {
"id": 3,
"uuid": "638828d7-7145-4060-904c-894aa726b133",
"name": 'fake_xilinx_fpga_dp',
"description": "fake_xilinx_fpga_dp-desc",
"created_at": datetime.datetime(
2022, 1, 22, 10, 40, 56,
tzinfo=datetime.timezone.utc),
"updated_at": None,
"groups": [
{"resources:FPGA": "1",
"trait:CUSTOM_FPGA_XILINX": "required",
"trait:CUSTOM_FPGA_PRODUCT_ID_5000": "required",
},
]
}
return _convert_to_obj(xilinx_fpga_dp)

View File

@ -300,3 +300,47 @@ def get_patch_list(same_device=True):
'value': dev_uuid}
patch_list[newarq['uuid']] = [host_binding, inst_binding, dev_binding]
return patch_list, device_rp_uuid
def get_fake_xilinx_fpga_extarq_objs():
arqs = [
{"uuid": 'b8c19eb2-e03c-47b4-b7cf-ced6086b2d11',
"device_profile_group_id": 0,
"state": "Initial",
"device_profile_name": "fake_xilinx_fpga_dp",
"hostname": "myhost",
"instance_uuid": "5922a70f-1e06-4cfd-88dd-a332120d7144",
"attach_handle_type": "PCI",
# attach_handle info should vary across ARQs but ignored for testing
"attach_handle_info": {
"bus": "3b",
"device": "00",
"domain": "0000",
"function": "0"
},
"device_profile_group": {
"trait:CUSTOM_FPGA_XILINX": "required",
"resources:FPGA": "1",
"trait:CUSTOM_FPGA_PRODUCT_ID_5000": "required"}
},
{"uuid": '012955c7-90f9-45a9-bb7d-7c2907d8997f',
"device_profile_group_id": 1,
"state": "Initial",
"device_profile_name": "fake_xilinx_fpga_dp",
"hostname": "myhost",
"instance_uuid": "5922a70f-1e06-4cfd-88dd-a332120d7144",
"attach_handle_type": "PCI",
# attach_handle info should vary across ARQs but ignored for testing
"attach_handle_info": {
"bus": "3b",
"device": "00",
"domain": "0000",
"function": "1"
},
"device_profile_group": {
"trait:CUSTOM_FPGA_XILINX": "required",
"resources:FPGA": "1",
"trait:CUSTOM_FPGA_PRODUCT_ID_5000": "required"}
},
]
return list(map(_convert_from_dict_to_obj, arqs))

View File

@ -0,0 +1,6 @@
---
features:
- |
Add the Xilinx FPGA driver. Through this driver Cyborg can manage Xilinx
FPGA devices, including discovering devices' info and programming
``xclbin``.

View File

@ -49,6 +49,7 @@ cyborg.database.migration_backend =
cyborg.accelerator.driver =
intel_fpga_driver = cyborg.accelerator.drivers.fpga.intel.driver:IntelFPGADriver
inspur_fpga_driver = cyborg.accelerator.drivers.fpga.inspur.driver:InspurFPGADriver
xilinx_fpga_driver = cyborg.accelerator.drivers.fpga.xilinx.driver:XilinxFPGADriver
nvmf_spdk_driver = cyborg.accelerator.drivers.spdk.nvmf.nvmf:NVMFDRIVER
nvidia_gpu_driver = cyborg.accelerator.drivers.gpu.nvidia.driver:NVIDIAGPUDriver
fake_driver = cyborg.accelerator.drivers.fake:FakeDriver