Merge "Support sriovdp arbitrary resource names"
This commit is contained in:
Zuul
Gerrit Code Review
commit
b8423edd24
|
|
@ -75,8 +75,66 @@ In the above example two SR-IOV devices will be attached to pod. First one is de
|
|||
in sriov-net1 NetworkAttachmentDefinition, second one in sriov-net2. They may have
|
||||
different subnetId.
|
||||
|
||||
4. Specify resource names
|
||||
|
||||
The resource name *intel.com/sriov*, which used in the above example is the default
|
||||
resource name. This name was used in SR-IOV network device plugin in
|
||||
version 1 (release-v1 branch). But since latest version the device plugin can use any
|
||||
arbitrary name of the resources [3]_. This name should match "^\[a-zA-Z0-9\_\]+$"
|
||||
regular expression. To be able to work with arbitrary resource names
|
||||
physnet_resource_mappings and device_plugin_resource_prefix in [sriov] section
|
||||
of kuryr-controller configuration file should be filled. The default value for
|
||||
device_plugin_resource_prefix is intel.com, the same as in SR-IOV network device plugin,
|
||||
in case of SR-IOV network device plugin was started with value of -resource-prefix option
|
||||
different from intel.com, than value should be set to
|
||||
device_plugin_resource_prefix, otherwise kuryr-kubernetes will not work with resource.
|
||||
|
||||
Assume we have following SR-IOV network device plugin (defined by -config-file option)
|
||||
|
||||
.. code-block:: json
|
||||
|
||||
{
|
||||
"resourceList":
|
||||
[
|
||||
{
|
||||
"resourceName": "numa0",
|
||||
"rootDevices": ["0000:02:00.0"],
|
||||
"sriovMode": true,
|
||||
"deviceType": "netdevice"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
We defined numa0 resource name, also assume we started sriovdp with
|
||||
-resource-prefix samsung.com value. The PCI address of ens4f0 interface
|
||||
is "0000:02:00.0". If we assigned 8 VF to ens4f0 and launch SR-IOV network
|
||||
device plugin, we can see following state of kubernetes
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ kubectl get node node1 -o json | jq '.status.allocatable'
|
||||
{
|
||||
"cpu": "4",
|
||||
"ephemeral-storage": "269986638772",
|
||||
"hugepages-1Gi": "8Gi",
|
||||
"hugepages-2Mi": "0Gi",
|
||||
"samsung.com/numa0": "8",
|
||||
"memory": "7880620Ki",
|
||||
"pods": "1k"
|
||||
}
|
||||
|
||||
We have to add to the sriov section following mapping:
|
||||
|
||||
.. code-block:: ini
|
||||
|
||||
[sriov]
|
||||
device_plugin_resource_prefix = samsung.com
|
||||
physnet_resource_mappings = physnet1:numa0
|
||||
|
||||
|
||||
Reference
|
||||
---------
|
||||
|
||||
.. [1] https://docs.openstack.org/kuryr-kubernetes/latest/specs/rocky/npwg_spec_support.html
|
||||
.. [2] https://docs.google.com/document/d/1D3dJeUUmta3sMzqw8JtWFoG2rvcJiWitVro9bsfUTEw
|
||||
.. [3] https://github.com/intel/sriov-network-device-plugin
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from kuryr.lib import config as lib_config
|
|||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
|
||||
from kuryr_kubernetes import constants
|
||||
from kuryr_kubernetes import version
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
|
@ -260,6 +261,18 @@ sriov_opts = [
|
|||
"physical network names to the agent's node-specific "
|
||||
"physical network device interfaces of SR-IOV physical "
|
||||
"function to be used for VLAN networks.")),
|
||||
cfg.DictOpt('physnet_resource_mappings',
|
||||
help=_("A mapping of physnets for certain sriov dp "
|
||||
"resource name in a form of "
|
||||
"physnet-name:resource name. "
|
||||
"Resource name is listed in sriov device plugin "
|
||||
"configuation file."),
|
||||
default=DEFAULT_PHYSNET_SUBNET_MAPPINGS),
|
||||
cfg.StrOpt('device_plugin_resource_prefix',
|
||||
help=_("This prefix is used by sriov-network-device-plugin "
|
||||
"It concatenates with resource suffix defined in "
|
||||
"sriov device plugin configuration file."),
|
||||
default=constants.K8S_SRIOV_PREFIX),
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ VIF_POOL_SHOW = '/showPool'
|
|||
DEFAULT_IFNAME = 'eth0'
|
||||
|
||||
ADDITIONAL_IFNAME_PREFIX = 'eth'
|
||||
K8S_NPWG_SRIOV_PREFIX = "intel.com/sriov"
|
||||
K8S_SRIOV_PREFIX = 'intel.com'
|
||||
|
||||
K8S_OPERATOR_IN = 'in'
|
||||
K8S_OPERATOR_NOT_IN = 'notin'
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@ from oslo_log import log as logging
|
|||
|
||||
from kuryr_kubernetes import clients
|
||||
from kuryr_kubernetes import config
|
||||
from kuryr_kubernetes import constants
|
||||
from kuryr_kubernetes.controller.drivers import neutron_vif
|
||||
from kuryr_kubernetes.controller.drivers import utils as c_utils
|
||||
from kuryr_kubernetes import os_vif_util as ovu
|
||||
|
|
@ -25,27 +24,33 @@ from kuryr_kubernetes import os_vif_util as ovu
|
|||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def sriov_make_resource(prefix, res_name):
|
||||
return prefix + "/" + res_name
|
||||
|
||||
|
||||
class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
||||
"""Provides VIFs for SRIOV VF interfaces."""
|
||||
|
||||
ALIAS = 'sriov_pod_vif'
|
||||
|
||||
def __init__(self):
|
||||
self._physnet_mapping = self._get_physnet_mapping()
|
||||
self._physnet_subnet_mapping = self._get_physnet_subnet_mapping()
|
||||
self._physnet_resname_mapping = self._get_physnet_resname_mapping()
|
||||
self._res_prefix = config.CONF.sriov.device_plugin_resource_prefix
|
||||
|
||||
def request_vif(self, pod, project_id, subnets, security_groups):
|
||||
amount = self._get_remaining_sriov_vfs(pod)
|
||||
if not amount:
|
||||
LOG.error("SRIOV VIF request failed due to lack of "
|
||||
"available VFs for the current pod creation")
|
||||
return None
|
||||
|
||||
pod_name = pod['metadata']['name']
|
||||
neutron = clients.get_neutron_client()
|
||||
vif_plugin = 'sriov'
|
||||
subnet_id = next(iter(subnets))
|
||||
physnet = self._get_physnet_for_subnet_id(subnet_id)
|
||||
LOG.debug("Pod {} handling {}".format(pod_name, physnet))
|
||||
|
||||
amount = self._get_remaining_sriov_vfs(pod, physnet)
|
||||
if not amount:
|
||||
LOG.error("SRIOV VIF request failed due to lack of "
|
||||
"available VFs for the current pod creation")
|
||||
return None
|
||||
rq = self._get_port_request(pod, project_id,
|
||||
subnets, security_groups)
|
||||
|
||||
|
|
@ -57,13 +62,13 @@ class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
|||
LOG.debug("{} vifs are available for the pod {}".format(
|
||||
amount, pod_name))
|
||||
|
||||
self._reduce_remaining_sriov_vfs(pod)
|
||||
self._reduce_remaining_sriov_vfs(pod, physnet)
|
||||
return vif
|
||||
|
||||
def activate_vif(self, pod, vif):
|
||||
vif.active = True
|
||||
|
||||
def _get_physnet_mapping(self):
|
||||
def _get_physnet_subnet_mapping(self):
|
||||
physnets = config.CONF.sriov.default_physnet_subnets
|
||||
|
||||
result = {}
|
||||
|
|
@ -71,17 +76,29 @@ class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
|||
result[subnet_id] = name
|
||||
return result
|
||||
|
||||
def _get_physnet_resname_mapping(self):
|
||||
resources = config.CONF.sriov.physnet_resource_mappings
|
||||
result = {}
|
||||
|
||||
if resources:
|
||||
for physnet_name, resource_name in resources.items():
|
||||
result[physnet_name] = resource_name
|
||||
else:
|
||||
for k, v in self._physnet_subnet_mapping.items():
|
||||
result[v] = v
|
||||
return result
|
||||
|
||||
def _get_physnet_for_subnet_id(self, subnet_id):
|
||||
"""Returns an appropriate physnet for exact subnet_id from mapping"""
|
||||
try:
|
||||
physnet = self._physnet_mapping[subnet_id]
|
||||
physnet = self._physnet_subnet_mapping[subnet_id]
|
||||
except KeyError:
|
||||
LOG.error("No mapping for subnet {} in {}".format(
|
||||
subnet_id, self._physnet_mapping))
|
||||
subnet_id, self._physnet_subnet_mapping))
|
||||
raise
|
||||
return physnet
|
||||
|
||||
def _get_remaining_sriov_vfs(self, pod):
|
||||
def _get_remaining_sriov_vfs(self, pod, physnet):
|
||||
"""Returns the number of remaining vfs.
|
||||
|
||||
Returns the number of remaining vfs from the initial number that
|
||||
|
|
@ -90,27 +107,42 @@ class SriovVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
|||
"""
|
||||
containers = pod['spec']['containers']
|
||||
total_amount = 0
|
||||
|
||||
sriov_resource_name = self._physnet_resname_mapping.get(physnet, None)
|
||||
if not sriov_resource_name:
|
||||
LOG.error("No mapping for physnet {} in {}".format(
|
||||
physnet, self._physnet_resname_mapping))
|
||||
return 0
|
||||
sriov_resource_name = sriov_make_resource(self._res_prefix,
|
||||
sriov_resource_name)
|
||||
for container in containers:
|
||||
try:
|
||||
requests = container['resources']['requests']
|
||||
amount_value = requests[constants.K8S_NPWG_SRIOV_PREFIX]
|
||||
amount_value = requests[sriov_resource_name]
|
||||
total_amount += int(amount_value)
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
return total_amount
|
||||
|
||||
def _reduce_remaining_sriov_vfs(self, pod):
|
||||
def _reduce_remaining_sriov_vfs(self, pod, physnet):
|
||||
"""Reduces number of available vfs for request"""
|
||||
|
||||
sriov_resource_name = self._physnet_resname_mapping.get(physnet, None)
|
||||
if not sriov_resource_name:
|
||||
LOG.error("No mapping for physnet {} in {}".format(
|
||||
physnet, self._physnet_resname_mapping))
|
||||
return
|
||||
containers = pod['spec']['containers']
|
||||
sriov_resource_name = sriov_make_resource(self._res_prefix,
|
||||
sriov_resource_name)
|
||||
for container in containers:
|
||||
try:
|
||||
requests = container['resources']['requests']
|
||||
num_of_sriov = int(requests[constants.K8S_NPWG_SRIOV_PREFIX])
|
||||
num_of_sriov = int(requests[sriov_resource_name])
|
||||
if num_of_sriov == 0:
|
||||
continue
|
||||
requests[constants.K8S_NPWG_SRIOV_PREFIX] = (
|
||||
str(num_of_sriov - 1))
|
||||
requests[sriov_resource_name] = str(num_of_sriov - 1)
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@
|
|||
import mock
|
||||
import uuid
|
||||
|
||||
from kuryr_kubernetes.controller.drivers import sriov as sriov_drivers
|
||||
from kuryr_kubernetes.controller.drivers import sriov as drvs
|
||||
from kuryr_kubernetes.tests import base as test_base
|
||||
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
|
||||
|
||||
|
|
@ -26,10 +26,30 @@ from kuryr_kubernetes import utils
|
|||
from oslo_config import cfg as oslo_cfg
|
||||
|
||||
|
||||
SRIOV_RESOURCE_NAME_A = "sriov_a"
|
||||
SRIOV_RESOURCE_NAME_B = "sriov_b"
|
||||
|
||||
AMOUNT_FOR_SUBNET_A = 2
|
||||
AMOUNT_FOR_SUBNET_B = 3
|
||||
|
||||
SRIOV_PHYSNET_A = "physnet_a"
|
||||
SRIOV_PHYSNET_B = "physnet_b"
|
||||
|
||||
|
||||
class TestSriovVIFDriver(test_base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSriovVIFDriver, self).setUp()
|
||||
|
||||
self._res_map = {SRIOV_PHYSNET_A: SRIOV_RESOURCE_NAME_A,
|
||||
SRIOV_PHYSNET_B: SRIOV_RESOURCE_NAME_B}
|
||||
sriov_request = {drvs.sriov_make_resource(k_const.K8S_SRIOV_PREFIX,
|
||||
SRIOV_RESOURCE_NAME_A): (
|
||||
str(AMOUNT_FOR_SUBNET_A)),
|
||||
drvs.sriov_make_resource(k_const.K8S_SRIOV_PREFIX,
|
||||
SRIOV_RESOURCE_NAME_B): (
|
||||
str(AMOUNT_FOR_SUBNET_B))}
|
||||
|
||||
self._pod = {
|
||||
'metadata': {
|
||||
'resourceVersion': mock.sentinel.pod_version,
|
||||
|
|
@ -41,16 +61,14 @@ class TestSriovVIFDriver(test_base.TestCase):
|
|||
'nodeName': 'hostname',
|
||||
'containers': [{
|
||||
'resources': {
|
||||
'requests': {
|
||||
k_const.K8S_NPWG_SRIOV_PREFIX: "2"
|
||||
}
|
||||
'requests': sriov_request
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
def test_activate_vif(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
pod = mock.sentinel.pod
|
||||
|
|
@ -63,7 +81,7 @@ class TestSriovVIFDriver(test_base.TestCase):
|
|||
@mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips')
|
||||
@mock.patch.object(ovu, 'neutron_to_osvif_vif')
|
||||
def test_request_vif(self, m_to_vif, m_to_fips):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||
|
|
@ -95,7 +113,7 @@ class TestSriovVIFDriver(test_base.TestCase):
|
|||
@mock.patch('kuryr_kubernetes.os_vif_util.osvif_to_neutron_fixed_ips')
|
||||
@mock.patch.object(ovu, 'neutron_to_osvif_vif')
|
||||
def test_request_vif_not_enough_vfs(self, m_to_vif, m_to_fips):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
m_driver._get_remaining_sriov_vfs.return_value = 0
|
||||
|
|
@ -112,22 +130,38 @@ class TestSriovVIFDriver(test_base.TestCase):
|
|||
neutron.create_port.assert_not_called()
|
||||
|
||||
def test_get_sriov_num_vf(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod)
|
||||
self.assertEqual(amount, 2)
|
||||
m_driver._physnet_resname_mapping = self._res_map
|
||||
m_driver._res_prefix = k_const.K8S_SRIOV_PREFIX
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod,
|
||||
SRIOV_PHYSNET_A)
|
||||
self.assertEqual(amount, AMOUNT_FOR_SUBNET_A)
|
||||
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod,
|
||||
SRIOV_PHYSNET_B)
|
||||
self.assertEqual(amount, AMOUNT_FOR_SUBNET_B)
|
||||
|
||||
def test_reduce_remaining_sriov_vfs(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
cls._reduce_remaining_sriov_vfs(m_driver, self._pod)
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod)
|
||||
self.assertEqual(amount, 1)
|
||||
m_driver._physnet_resname_mapping = self._res_map
|
||||
m_driver._res_prefix = k_const.K8S_SRIOV_PREFIX
|
||||
|
||||
def test_get_physnet_mapping(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls._reduce_remaining_sriov_vfs(m_driver, self._pod, SRIOV_PHYSNET_A)
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod,
|
||||
SRIOV_PHYSNET_A)
|
||||
self.assertEqual(amount, AMOUNT_FOR_SUBNET_A - 1)
|
||||
|
||||
cls._reduce_remaining_sriov_vfs(m_driver, self._pod, SRIOV_PHYSNET_B)
|
||||
amount = cls._get_remaining_sriov_vfs(m_driver, self._pod,
|
||||
SRIOV_PHYSNET_B)
|
||||
self.assertEqual(amount, AMOUNT_FOR_SUBNET_B - 1)
|
||||
|
||||
def test_get_physnet_subnet_mapping(self):
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
subnet_id = str(uuid.uuid4())
|
||||
|
|
@ -135,25 +169,52 @@ class TestSriovVIFDriver(test_base.TestCase):
|
|||
'physnet10_4:'+str(subnet_id),
|
||||
group='sriov')
|
||||
|
||||
mapping = cls._get_physnet_mapping(m_driver)
|
||||
mapping = cls._get_physnet_subnet_mapping(m_driver)
|
||||
self.assertEqual(mapping, {subnet_id: 'physnet10_4'})
|
||||
|
||||
def test_get_physnet_resname_mapping(self):
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
oslo_cfg.CONF.set_override('physnet_resource_mappings',
|
||||
SRIOV_PHYSNET_A + ':' +
|
||||
SRIOV_RESOURCE_NAME_A + ',' +
|
||||
SRIOV_PHYSNET_B + ':' +
|
||||
SRIOV_RESOURCE_NAME_B,
|
||||
group='sriov')
|
||||
|
||||
mapping = cls._get_physnet_resname_mapping(m_driver)
|
||||
self.assertEqual(mapping, self._res_map)
|
||||
|
||||
def test_empty_physnet_resname_mapping(self):
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
empty_res_map = {SRIOV_PHYSNET_A: SRIOV_PHYSNET_A,
|
||||
SRIOV_PHYSNET_B: SRIOV_PHYSNET_B}
|
||||
subnet_id = str(uuid.uuid4())
|
||||
subnet_id_2 = str(uuid.uuid4())
|
||||
m_driver._physnet_subnet_mapping = {subnet_id: SRIOV_PHYSNET_A,
|
||||
subnet_id_2: SRIOV_PHYSNET_B}
|
||||
mapping = cls._get_physnet_resname_mapping(m_driver)
|
||||
self.assertEqual(mapping, empty_res_map)
|
||||
|
||||
def test_get_physnet_for_subnet_id(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
subnet_id = str(uuid.uuid4())
|
||||
m_driver._physnet_mapping = {subnet_id: 'physnet10_4'}
|
||||
m_driver._physnet_subnet_mapping = {subnet_id: 'physnet10_4'}
|
||||
|
||||
physnet = cls._get_physnet_for_subnet_id(m_driver, subnet_id)
|
||||
self.assertEqual(physnet, 'physnet10_4')
|
||||
|
||||
def test_get_physnet_for_subnet_id_error(self):
|
||||
cls = sriov_drivers.SriovVIFDriver
|
||||
cls = drvs.SriovVIFDriver
|
||||
m_driver = mock.Mock(spec=cls)
|
||||
|
||||
subnet_id = str(uuid.uuid4())
|
||||
m_driver._physnet_mapping = {}
|
||||
m_driver._physnet_subnet_mapping = {}
|
||||
|
||||
self.assertRaises(KeyError, cls._get_physnet_for_subnet_id,
|
||||
m_driver, subnet_id)
|
||||
|
|
|
|||
Loading…
Reference in New Issue