Add NVMe SSD driver

This patch implemented NVMe SSD driver in Cyborg.
The Inspur NVMe SSD driver provides the discover and report proposal of
Inspur NVMe SSD disks, then we can use these disks binding and unbinding
with VM like PGPU to accelerator the io rate for the VM. The Inspur NVMe
SSD doesnot support virtualization, one disk can be only bind to one VM.

The spec is already commit. Please see:
https://specs.openstack.org/openstack/cyborg-specs/specs/wallaby/approved/nvme-ssd-driver-proposal.html

Please check the test report in the following link:
https://wiki.openstack.org/wiki/Cyborg/TestReport/InspurNVMeSSD

Co-Authored-By: Wenping Song <songwenping@inspur.com>

Change-Id: Ic474814b780e9beca6f19df50c9ce4c5553850a1
changes/10/723310/14
zhangbailin 3 years ago committed by songwenping
parent 82d1ed968b
commit 19b7d147ea
  1. 14
      cyborg/accelerator/common/utils.py
  2. 0
      cyborg/accelerator/drivers/ssd/__init__.py
  3. 62
      cyborg/accelerator/drivers/ssd/base.py
  4. 0
      cyborg/accelerator/drivers/ssd/inspur/__init__.py
  5. 28
      cyborg/accelerator/drivers/ssd/inspur/driver.py
  6. 27
      cyborg/accelerator/drivers/ssd/inspur/sysinfo.py
  7. 172
      cyborg/accelerator/drivers/ssd/utils.py
  8. 11
      cyborg/common/constants.py
  9. 22
      cyborg/db/sqlalchemy/alembic/versions/4cc1d79978fc_add_ssd_type.py
  10. 2
      cyborg/db/sqlalchemy/models.py
  11. 0
      cyborg/tests/unit/accelerator/drivers/ssd/__init__.py
  12. 0
      cyborg/tests/unit/accelerator/drivers/ssd/inspur/__init__.py
  13. 32
      cyborg/tests/unit/accelerator/drivers/ssd/test_base.py
  14. 137
      cyborg/tests/unit/accelerator/drivers/ssd/test_utils.py
  15. 6
      doc/source/reference/driver-table.rst
  16. 7
      releasenotes/notes/inspur-nvme-ssd-faeddc0b09250acc.yaml
  17. 1
      setup.cfg

@ -92,3 +92,17 @@ def parse_mappings(mapping_list):
mapping[physnet_or_function] = set(dev.strip() for dev in
devices.split("|") if dev.strip())
return mapping
def get_vendor_maps():
"""The data is based on http://pci-ids.ucw.cz/read/PC/
:return: vendor maps dict
"""
return {"10de": "nvidia",
"102b": "matrox",
"1bd4": "inspur",
"8086": "intel",
"1099": "samsung",
"1cf2": "zte"
}

@ -0,0 +1,62 @@
# Copyright 2020 Inspur Electronic Information Industry Co.,LTD.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg Generic SSD driver implementation.
"""
from oslo_log import log as logging
from cyborg.accelerator.common import utils as pci_utils
from cyborg.accelerator.drivers.ssd import utils
LOG = logging.getLogger(__name__)
VENDOR_MAPS = pci_utils.get_vendor_maps()
class SSDDriver(object):
"""Generic class for SSD drivers.
This is just a virtual SSD drivers interface.
Vendor should implement their specific drivers.
"""
@classmethod
def create(cls, vendor=None, *args, **kwargs):
if not vendor:
return cls(*args, **kwargs)
for sclass in cls.__subclasses__():
vendor_name = VENDOR_MAPS.get(vendor, vendor)
if vendor_name == sclass.VENDOR:
return sclass(*args, **kwargs)
raise LookupError("Not find the SSD driver for vendor %s" % vendor)
def discover(self):
"""Discover SSD information of current vendor(Identified by class).
If no vendor-specific driver provided, this will return all NVMe SSD
devices
:return: List of SSD information dict.
"""
LOG.info('The method "discover" is called in generic.SSDDriver')
devs = utils.discover_ssds()
return devs
@classmethod
def discover_vendors(cls):
"""Discover SSD vendors of current node.
:return: SSD vendor ID list.
"""
return utils.discover_vendors()

@ -0,0 +1,28 @@
# Copyright 2020 Inspur Electronic Information Industry Co.,LTD.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg Inspur NVMe SSD driver implementation.
"""
from cyborg.accelerator.drivers.ssd.base import SSDDriver
from cyborg.accelerator.drivers.ssd.inspur import sysinfo
class InspurNVMeSSDDriver(SSDDriver):
VENDOR = "inspur"
def discover(self):
return sysinfo.nvme_ssd_tree()

@ -0,0 +1,27 @@
# Copyright 2020 Inspur Electronic Information Industry Co.,LTD.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Cyborg Inspur NVMe SSD driver implementation.
"""
from cyborg.accelerator.drivers.ssd import utils
VENDOR_ID = '1bd4'
def nvme_ssd_tree():
devs = utils.discover_ssds(VENDOR_ID)
return devs

@ -0,0 +1,172 @@
# Copyright 2020 Inspur Electronic Information Industry Co.,LTD.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Utils for SSD driver.
"""
from oslo_concurrency import processutils
from oslo_log import log as logging
from oslo_serialization import jsonutils
import re
from cyborg.accelerator.common import utils
from cyborg.common import constants
from cyborg.conf import CONF
from cyborg.objects.driver_objects import driver_attach_handle
from cyborg.objects.driver_objects import driver_attribute
from cyborg.objects.driver_objects import driver_controlpath_id
from cyborg.objects.driver_objects import driver_deployable
from cyborg.objects.driver_objects import driver_device
import cyborg.privsep
LOG = logging.getLogger(__name__)
SSD_FLAGS = ["Non-Volatile memory controller"]
SSD_INFO_PATTERN = re.compile(r"(?P<devices>[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:"
r"[0-9a-fA-F]{2}\.[0-9a-fA-F]) "
r"(?P<controller>.*) [\[].*]: (?P<model>.*) .*"
r"[\[](?P<vendor_id>[0-9a-fA-F]"
r"{4}):(?P<product_id>[0-9a-fA-F]{4})].*")
VENDOR_MAPS = utils.get_vendor_maps()
@cyborg.privsep.sys_admin_pctxt.entrypoint
def lspci_privileged():
cmd = ['lspci', '-nn', '-D']
return processutils.execute(*cmd)
def get_pci_devices(pci_flags, vendor_id=None):
device_for_vendor_out = []
all_device_out = []
lspci_out = lspci_privileged()[0].split('\n')
for i in range(len(lspci_out)):
if any(x in lspci_out[i] for x in pci_flags):
all_device_out.append(lspci_out[i])
if vendor_id and vendor_id in lspci_out[i]:
device_for_vendor_out.append(lspci_out[i])
return device_for_vendor_out if vendor_id else all_device_out
def get_traits(vendor_id, product_id):
"""Generate traits for SSDs.
: param vendor_id: vendor_id of SSD, eg."1bd4"
: param product_id: product_id of SSD, eg."1003".
Example SSD traits:
{traits:["CUSTOM_SSD_INSPUR", "CUSTOM_SSD_PRODUCT_ID_1003"]}
"""
traits = []
traits.append("CUSTOM_SSD_" + VENDOR_MAPS.get(vendor_id, "").upper())
traits.append("CUSTOM_SSD_PRODUCT_ID_" + product_id.upper())
return {"traits": traits}
def discover_vendors():
vendors = set()
ssds = get_pci_devices(SSD_FLAGS)
for ssd in ssds:
m = SSD_INFO_PATTERN.match(ssd)
if m:
vendor_id = m.groupdict().get("vendor_id")
vendors.add(vendor_id)
return vendors
def discover_ssds(vendor_id=None):
ssd_list = []
ssds = get_pci_devices(SSD_FLAGS, vendor_id)
for ssd in ssds:
m = SSD_INFO_PATTERN.match(ssd)
if m:
ssd_dict = m.groupdict()
ssd_dict['hostname'] = CONF.host
# generate traits info
traits = get_traits(ssd_dict["vendor_id"], ssd_dict["product_id"])
ssd_dict["rc"] = constants.RESOURCES["SSD"]
ssd_dict.update(traits)
ssd_list.append(_generate_driver_device(ssd_dict))
return ssd_list
def _generate_driver_device(ssd):
driver_device_obj = driver_device.DriverDevice()
driver_device_obj.vendor = ssd["vendor_id"]
driver_device_obj.model = ssd.get('model', 'miss model info')
std_board_info = {'product_id': ssd.get('product_id', None),
'controller': ssd.get('controller', None)}
vendor_board_info = {'vendor_info': ssd.get('vendor_info', 'ssd_vb_info')}
driver_device_obj.std_board_info = jsonutils.dumps(std_board_info)
driver_device_obj.vendor_board_info = jsonutils.dumps(vendor_board_info)
driver_device_obj.type = constants.DEVICE_SSD
driver_device_obj.stub = ssd.get('stub', False)
driver_device_obj.controlpath_id = _generate_controlpath_id(ssd)
driver_device_obj.deployable_list = _generate_dep_list(ssd)
return driver_device_obj
def _generate_controlpath_id(ssd):
driver_cpid = driver_controlpath_id.DriverControlPathID()
# NOTE: SSDs , they all report "PCI" as
# their cpid_type, while attach_handle_type of them are different.
driver_cpid.cpid_type = "PCI"
driver_cpid.cpid_info = utils.pci_str_to_json(ssd["devices"])
return driver_cpid
def _generate_dep_list(ssd):
dep_list = []
driver_dep = driver_deployable.DriverDeployable()
driver_dep.attribute_list = _generate_attribute_list(ssd)
driver_dep.attach_handle_list = []
# NOTE(wenping) Now simply named as <Compute_hostname>_<Device_address>
# once cyborg needs to support SSD devices discovered from a baremetal
# node, we might need to support more formats.
driver_dep.name = ssd.get('hostname', '') + '_' + ssd["devices"]
driver_dep.driver_name = VENDOR_MAPS.get(ssd["vendor_id"]).upper()
# driver_dep.num_accelerators for SSD is 1
driver_dep.num_accelerators = 1
driver_dep.attach_handle_list = [_generate_attach_handle(ssd)]
dep_list.append(driver_dep)
return dep_list
def _generate_attach_handle(ssd):
driver_ah = driver_attach_handle.DriverAttachHandle()
if ssd["rc"] == "CUSTOM_SSD":
driver_ah.attach_type = constants.AH_TYPE_PCI
else:
driver_ah.attach_type = constants.AH_TYPE_MDEV
driver_ah.in_use = False
driver_ah.attach_info = utils.pci_str_to_json(ssd["devices"])
return driver_ah
def _generate_attribute_list(ssd):
attr_list = []
for k, v in ssd.items():
if k == "rc":
driver_attr = driver_attribute.DriverAttribute()
driver_attr.key, driver_attr.value = k, v
attr_list.append(driver_attr)
if k == "traits":
values = ssd.get(k, [])
for index, val in enumerate(values):
driver_attr = driver_attribute.DriverAttribute(
key="trait" + str(index), value=val)
attr_list.append(driver_attr)
return attr_list

@ -21,6 +21,7 @@ DEVICE_FPGA = 'FPGA'
DEVICE_AICHIP = 'AICHIP'
DEVICE_QAT = 'QAT'
DEVICE_NIC = 'NIC'
DEVICE_SSD = 'SSD'
ARQ_STATES = (ARQ_INITIAL, ARQ_BIND_STARTED, ARQ_BOUND, ARQ_UNBOUND,
@ -59,7 +60,8 @@ ARQ_STATES_TRANSFORM_MATRIX = {
# Device type
DEVICE_TYPE = (DEVICE_GPU, DEVICE_FPGA, DEVICE_AICHIP, DEVICE_QAT, DEVICE_NIC)
DEVICE_TYPE = (DEVICE_GPU, DEVICE_FPGA, DEVICE_AICHIP, DEVICE_QAT, DEVICE_NIC,
DEVICE_SSD)
# Attach handle type
@ -78,7 +80,8 @@ RESOURCES = {
"PGPU": orc.PGPU,
"VGPU": orc.VGPU,
"QAT": "CUSTOM_QAT",
"NIC": "CUSTOM_NIC"
"NIC": "CUSTOM_NIC",
"SSD": 'CUSTOM_SSD',
}
@ -92,8 +95,8 @@ ACCEL_SPECS = (
SUPPORT_RESOURCES = (
FPGA, GPU, VGPU, PGPU, QAT, NIC) = (
"FPGA", "GPU", "VGPU", "PGPU", "CUSTOM_QAT", "CUSTOM_NIC"
FPGA, GPU, VGPU, PGPU, QAT, NIC, SSD) = (
"FPGA", "GPU", "VGPU", "PGPU", "CUSTOM_QAT", "CUSTOM_NIC", "CUSTOM_SSD"
)

@ -0,0 +1,22 @@
"""add_ssd_type
Revision ID: 4cc1d79978fc
Revises: 899cead40bc9
Create Date: 2021-02-15 16:02:58.856126
"""
# revision identifiers, used by Alembic.
revision = '4cc1d79978fc'
down_revision = '899cead40bc9'
from alembic import op
import sqlalchemy as sa
def upgrade():
new_device_type = sa.Enum('GPU', 'FPGA', 'AICHIP', 'QAT', 'NIC', 'SSD',
name='device_type')
op.alter_column('devices', 'type',
existing_type=new_device_type,
nullable=False)

@ -81,7 +81,7 @@ class Device(Base):
id = Column(Integer, primary_key=True)
uuid = Column(String(36), nullable=False, unique=True)
type = Column(Enum('GPU', 'FPGA', 'AICHIP', 'QAT', 'NIC',
type = Column(Enum('GPU', 'FPGA', 'AICHIP', 'QAT', 'NIC', 'SSD',
name='device_type'), nullable=False)
vendor = Column(String(255), nullable=False)
model = Column(String(255), nullable=False)

@ -0,0 +1,32 @@
# Copyright 2020 Inspur Electronic Information Industry Co.,LTD..
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from cyborg.accelerator.drivers.ssd.base import SSDDriver
from cyborg.accelerator.drivers.ssd.inspur.driver import InspurNVMeSSDDriver
from cyborg.tests import base
class TestSSDDriver(base.TestCase):
def test_create_base_ssd_driver(self):
drv = SSDDriver.create()
self.assertIsInstance(drv, SSDDriver)
self.assertNotIsInstance(drv, InspurNVMeSSDDriver)
def test_create_inspur_ssd_driver(self):
drv = SSDDriver.create(vendor='inspur')
self.assertIsInstance(drv, InspurNVMeSSDDriver)
def test_create_ssd_vendor_not_found(self):
self.assertRaises(LookupError, SSDDriver.create, '_non-exist_vendor')

@ -0,0 +1,137 @@
# Copyright 2020 Inspur Electronic Information Industry Co.,LTD.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest import mock
from oslo_serialization import jsonutils
from cyborg.accelerator.drivers.ssd.base import SSDDriver
from cyborg.accelerator.drivers.ssd.inspur.driver import InspurNVMeSSDDriver
from cyborg.accelerator.drivers.ssd import utils
from cyborg.tests import base
NVME_SSD_INFO = \
"0000:db:00.0 Non-Volatile memory controller [0108]: Inspur " \
"Electronic Information Industry Co., Ltd. Device [1bd4:1001]" \
" (rev 02)\n0000:db:01.0 Non-Volatile memory controller " \
"[0108]: Inspur Electronic Information Industry Co., Ltd. " \
"Device [1bd4:1001] (rev 02)"
class stdout(object):
def readlines(self):
return [NVME_SSD_INFO]
class p(object):
def __init__(self):
self.stdout = stdout()
def wait(self):
pass
class TestSSDDriverUtils(base.TestCase):
def setUp(self):
super().setUp()
self.p = p()
@mock.patch('cyborg.accelerator.drivers.ssd.utils.lspci_privileged')
def test_discover_vendors(self, mock_devices):
mock_devices.return_value = self.p.stdout.readlines()
ssd_vendors = utils.discover_vendors()
self.assertEqual(1, len(ssd_vendors))
@mock.patch('cyborg.accelerator.drivers.ssd.utils.lspci_privileged')
def test_discover_with_inspur_ssd_driver(self, mock_devices_for_vendor):
mock_devices_for_vendor.return_value = self.p.stdout.readlines()
self.set_defaults(host='host-192-168-32-195', debug=True)
vendor_id = '1bd4'
ssd_list = InspurNVMeSSDDriver.discover(vendor_id)
self.assertEqual(2, len(ssd_list))
attach_handle_list = [
{'attach_type': 'PCI',
'attach_info': '{"bus": "db", '
'"device": "00", '
'"domain": "0000", '
'"function": "0"}',
'in_use': False}
]
attribute_list = [
{'key': 'rc', 'value': 'CUSTOM_SSD'},
{'key': 'trait0', 'value': 'CUSTOM_SSD_INSPUR'},
{'key': 'trait1', 'value': 'CUSTOM_SSD_PRODUCT_ID_1001'},
]
expected = {
'vendor': '1bd4',
'type': 'SSD',
'std_board_info':
{"controller": "Non-Volatile memory controller",
"product_id": "1001"},
'vendor_board_info': {"vendor_info": "ssd_vb_info"},
'deployable_list':
[
{
'num_accelerators': 1,
'driver_name': 'INSPUR',
'name': 'host-192-168-32-195_0000:db:00.0',
'attach_handle_list': attach_handle_list,
'attribute_list': attribute_list
},
],
'controlpath_id': {'cpid_info': '{"bus": "db", '
'"device": "00", '
'"domain": "0000", '
'"function": "0"}',
'cpid_type': 'PCI'}
}
ssd_obj = ssd_list[0]
ssd_dict = ssd_obj.as_dict()
ssd_dep_list = ssd_dict['deployable_list']
ssd_attach_handle_list = \
ssd_dep_list[0].as_dict()['attach_handle_list']
ssd_attribute_list = \
ssd_dep_list[0].as_dict()['attribute_list']
attri_obj_data = []
[attri_obj_data.append(attr.as_dict()) for attr in ssd_attribute_list]
attribute_actual_data = sorted(attri_obj_data, key=lambda i: i['key'])
self.assertEqual(expected['vendor'], ssd_dict['vendor'])
self.assertEqual(expected['controlpath_id'],
ssd_dict['controlpath_id'])
self.assertEqual(expected['std_board_info'],
jsonutils.loads(ssd_dict['std_board_info']))
self.assertEqual(expected['vendor_board_info'],
jsonutils.loads(ssd_dict['vendor_board_info']))
self.assertEqual(expected['deployable_list'][0]['num_accelerators'],
ssd_dep_list[0].as_dict()['num_accelerators'])
self.assertEqual(expected['deployable_list'][0]['name'],
ssd_dep_list[0].as_dict()['name'])
self.assertEqual(expected['deployable_list'][0]['driver_name'],
ssd_dep_list[0].as_dict()['driver_name'])
self.assertEqual(attach_handle_list[0],
ssd_attach_handle_list[0].as_dict())
self.assertEqual(attribute_list, attribute_actual_data)
@mock.patch('cyborg.accelerator.drivers.ssd.utils.lspci_privileged')
def test_discover_with_base_ssd_driver(self, mock_devices_for_vendor):
mock_devices_for_vendor.return_value = self.p.stdout.readlines()
with self.assertLogs(None, level='INFO') as cm:
d = SSDDriver.create()
ssd_list = d.discover()
self.assertEqual(cm.output,
['INFO:cyborg.accelerator.drivers.ssd.base:The '
'method "discover" is called in generic.SSDDriver'])
self.assertEqual(2, len(ssd_list))
self.assertEqual("1bd4", ssd_list[1].as_dict()['vendor'])

@ -42,6 +42,12 @@
- The driver for Intel NIC Cards.
- None
- Test results reported at Feb 2021. Please reference: `Intel NIC Driver Test Report <https://wiki.openstack.org/wiki/Cyborg/TestReport/IntelNic>`_
* - Inspur NVMe SSD Driver
- None
- The driver for Inspur NVMe SSD DISK.
- None
- Test results reported at Feb 2021. Please reference: `Inspur NVMe SSD Driver Test Report <https://wiki.openstack.org/wiki/Cyborg/TestReport/InspurNVMeSSD>`_
.. note:: Temporary Test Report: This is a temporary test report, it is only
valid for a short time, if you encounter problems, please contact the

@ -0,0 +1,7 @@
---
features:
- |
The Inspur NVMe SSD driver provides the discover and report proposal of
Inspur NVMe SSD disks, then we can use these disks binding and unbinding
with VM like PGPU to accelerator the io rate for the VM. The Inspur NVMe
SSD doesnot support virtualization, one disk can be only bind to one VM.

@ -53,6 +53,7 @@ cyborg.accelerator.driver =
huawei_ascend_driver = cyborg.accelerator.drivers.aichip.huawei.ascend:AscendDriver
intel_qat_driver = cyborg.accelerator.drivers.qat.intel.driver:IntelQATDriver
intel_nic_driver = cyborg.accelerator.drivers.nic.intel.driver:IntelNICDriver
inspur_nvme_ssd_driver = cyborg.accelerator.drivers.ssd.inspur.driver:InspurNVMeSSDDriver
oslo.config.opts =
cyborg = cyborg.conf.opts:list_opts

Loading…
Cancel
Save