Annotate nodes with pci info for direct ports
This commit allows to add pci information for direct neutron ports attached to PODs into nodes annotations. It happens on binding stage when pci information can be requested. Also this commit allows to delete annotations for such ports when apropriate POD is deleted and VFs are returned into host's network namespace. For future commits: it is necessary to update neutron ports with pci info when POD is in Running state. Change-Id: I5ea8da6bb3143fd689e701f5e503488d4a6c9b33 Closes-Bug: 1818606 Signed-off-by: Danil Golov <d.golov@samsung.com>
This commit is contained in:
parent
3791b84069
commit
b0ce30142e
|
@ -21,8 +21,10 @@ from oslo_concurrency import processutils
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from kuryr_kubernetes import clients
|
||||||
from kuryr_kubernetes.cni.binding import base as b_base
|
from kuryr_kubernetes.cni.binding import base as b_base
|
||||||
from kuryr_kubernetes import config
|
from kuryr_kubernetes import config
|
||||||
|
from kuryr_kubernetes import constants
|
||||||
from kuryr_kubernetes import exceptions
|
from kuryr_kubernetes import exceptions
|
||||||
from kuryr_kubernetes import utils
|
from kuryr_kubernetes import utils
|
||||||
|
|
||||||
|
@ -53,7 +55,7 @@ class VIFSriovDriver(object):
|
||||||
c_ipdb = b_base.get_ipdb(netns)
|
c_ipdb = b_base.get_ipdb(netns)
|
||||||
|
|
||||||
pf_names = self._get_host_pf_names(physnet)
|
pf_names = self._get_host_pf_names(physnet)
|
||||||
vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
|
vf_name, vf_index, pf, pci_info = self._get_available_vf_info(pf_names)
|
||||||
|
|
||||||
if not vf_name:
|
if not vf_name:
|
||||||
error_msg = "No free interfaces for physnet {} available".format(
|
error_msg = "No free interfaces for physnet {} available".format(
|
||||||
|
@ -75,11 +77,13 @@ class VIFSriovDriver(object):
|
||||||
iface.mtu = vif.network.mtu
|
iface.mtu = vif.network.mtu
|
||||||
iface.up()
|
iface.up()
|
||||||
|
|
||||||
|
self._save_pci_info(vif.id, pci_info)
|
||||||
|
|
||||||
def disconnect(self, vif, ifname, netns, container_id):
|
def disconnect(self, vif, ifname, netns, container_id):
|
||||||
# NOTE(k.zaitsev): when netns is deleted the interface is
|
# NOTE(k.zaitsev): when netns is deleted the interface is
|
||||||
# returned automatically to host netns. We may reset
|
# returned automatically to host netns. We may reset
|
||||||
# it to all-zero state
|
# it to all-zero state
|
||||||
pass
|
self._remove_pci_info(vif.id)
|
||||||
|
|
||||||
def _get_host_pf_names(self, physnet):
|
def _get_host_pf_names(self, physnet):
|
||||||
"""Return a list of PFs, that belong to a physnet"""
|
"""Return a list of PFs, that belong to a physnet"""
|
||||||
|
@ -120,9 +124,70 @@ class VIFSriovDriver(object):
|
||||||
self._release()
|
self._release()
|
||||||
continue
|
continue
|
||||||
vf_name = vf_names[0]
|
vf_name = vf_names[0]
|
||||||
|
pci_info = self._get_pci_info(pf, vf_index)
|
||||||
LOG.debug("Aquiring vf %s of pf %s", vf_index, pf)
|
LOG.debug("Aquiring vf %s of pf %s", vf_index, pf)
|
||||||
return vf_name, vf_index, pf
|
return vf_name, vf_index, pf, pci_info
|
||||||
return None, None, None
|
return None, None, None, None
|
||||||
|
|
||||||
|
def _get_pci_info(self, pf, vf_index):
|
||||||
|
pci_slot = ''
|
||||||
|
physnet = ''
|
||||||
|
pci_vendor_info = ''
|
||||||
|
|
||||||
|
vendor_path = '/sys/class/net/{}/device/virtfn{}/vendor'.format(
|
||||||
|
pf, vf_index)
|
||||||
|
with open(vendor_path) as vendor_file:
|
||||||
|
vendor_full = vendor_file.read()
|
||||||
|
vendor = vendor_full.split('x')[1].strip()
|
||||||
|
device_path = '/sys/class/net/{}/device/virtfn{}/device'.format(
|
||||||
|
pf, vf_index)
|
||||||
|
with open(device_path) as device_file:
|
||||||
|
device_full = device_file.read()
|
||||||
|
device = device_full.split('x')[1].strip()
|
||||||
|
pci_vendor_info = '{}:{}'.format(vendor, device)
|
||||||
|
|
||||||
|
vf_path = '/sys/class/net/{}/device/virtfn{}'.format(
|
||||||
|
pf, vf_index)
|
||||||
|
pci_slot_path = os.readlink(vf_path)
|
||||||
|
pci_slot = pci_slot_path.split('/')[1]
|
||||||
|
|
||||||
|
physnet = self._get_physnet_by_pf(pf)
|
||||||
|
|
||||||
|
return {'pci_slot': pci_slot,
|
||||||
|
'physical_network': physnet,
|
||||||
|
'pci_vendor_info': pci_vendor_info}
|
||||||
|
|
||||||
|
def _get_physnet_by_pf(self, desired_pf):
|
||||||
|
for physnet, pf_list in self._device_pf_mapping.items():
|
||||||
|
for pf in pf_list:
|
||||||
|
if pf == desired_pf:
|
||||||
|
return physnet
|
||||||
|
LOG.exception("Unable to find physnet for pf %s", desired_pf)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _save_pci_info(self, neutron_port, port_pci_info):
|
||||||
|
k8s = clients.get_kubernetes_client()
|
||||||
|
annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO
|
||||||
|
annot_name = annot_name.replace('/', '~1')
|
||||||
|
annot_name = annot_name + '-' + neutron_port
|
||||||
|
LOG.info('annot_name = %s', annot_name)
|
||||||
|
nodename = utils.get_node_name()
|
||||||
|
|
||||||
|
LOG.info("Trying to annotate node %s with pci info %s",
|
||||||
|
nodename, port_pci_info)
|
||||||
|
k8s.patch_node_annotations(nodename, annot_name, port_pci_info)
|
||||||
|
|
||||||
|
def _remove_pci_info(self, neutron_port):
|
||||||
|
k8s = clients.get_kubernetes_client()
|
||||||
|
annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO
|
||||||
|
annot_name = annot_name.replace('/', '~1')
|
||||||
|
annot_name = annot_name + '-' + neutron_port
|
||||||
|
LOG.info('annot_name = %s', annot_name)
|
||||||
|
nodename = utils.get_node_name()
|
||||||
|
|
||||||
|
LOG.info("Trying to delete pci info for port %s on node %s",
|
||||||
|
neutron_port, nodename)
|
||||||
|
k8s.remove_node_annotations(nodename, annot_name)
|
||||||
|
|
||||||
def _acquire(self, path):
|
def _acquire(self, path):
|
||||||
if self._lock and self._lock.acquired:
|
if self._lock and self._lock.acquired:
|
||||||
|
|
|
@ -52,6 +52,8 @@ K8S_ANNOTATION_NPWG_NETWORK = K8S_ANNOTATION_NPWG_PREFIX + '/networks'
|
||||||
K8S_ANNOTATION_NPWG_CRD_SUBNET_ID = 'subnetId'
|
K8S_ANNOTATION_NPWG_CRD_SUBNET_ID = 'subnetId'
|
||||||
K8S_ANNOTATION_NPWG_CRD_DRIVER_TYPE = 'driverType'
|
K8S_ANNOTATION_NPWG_CRD_DRIVER_TYPE = 'driverType'
|
||||||
|
|
||||||
|
K8S_ANNOTATION_NODE_PCI_DEVICE_INFO = 'openstack.org/kuryr-pci-info'
|
||||||
|
|
||||||
K8S_OS_VIF_NOOP_PLUGIN = "noop"
|
K8S_OS_VIF_NOOP_PLUGIN = "noop"
|
||||||
|
|
||||||
CNI_EXCEPTION_CODE = 100
|
CNI_EXCEPTION_CODE = 100
|
||||||
|
|
|
@ -22,6 +22,7 @@ import requests
|
||||||
|
|
||||||
from kuryr.lib._i18n import _
|
from kuryr.lib._i18n import _
|
||||||
from kuryr_kubernetes import config
|
from kuryr_kubernetes import config
|
||||||
|
from kuryr_kubernetes import constants
|
||||||
from kuryr_kubernetes import exceptions as exc
|
from kuryr_kubernetes import exceptions as exc
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
@ -125,6 +126,38 @@ class K8sClient(object):
|
||||||
return response.json().get('status')
|
return response.json().get('status')
|
||||||
raise exc.K8sClientException(response.text)
|
raise exc.K8sClientException(response.text)
|
||||||
|
|
||||||
|
def patch_node_annotations(self, node, annotation_name, value):
|
||||||
|
content_type = 'application/json-patch+json'
|
||||||
|
path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node)
|
||||||
|
value = jsonutils.dumps(value)
|
||||||
|
url, header = self._get_url_and_header(path, content_type)
|
||||||
|
|
||||||
|
data = [{'op': 'add',
|
||||||
|
'path': '/metadata/annotations/{}'.format(annotation_name),
|
||||||
|
'value': value}]
|
||||||
|
|
||||||
|
response = requests.patch(url, data=jsonutils.dumps(data),
|
||||||
|
headers=header, cert=self.cert,
|
||||||
|
verify=self.verify_server)
|
||||||
|
if response.ok:
|
||||||
|
return response.json().get('status')
|
||||||
|
raise exc.K8sClientException(response.text)
|
||||||
|
|
||||||
|
def remove_node_annotations(self, node, annotation_name):
|
||||||
|
content_type = 'application/json-patch+json'
|
||||||
|
path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node)
|
||||||
|
url, header = self._get_url_and_header(path, content_type)
|
||||||
|
|
||||||
|
data = [{'op': 'remove',
|
||||||
|
'path': '/metadata/annotations/{}'.format(annotation_name)}]
|
||||||
|
|
||||||
|
response = requests.patch(url, data=jsonutils.dumps(data),
|
||||||
|
headers=header, cert=self.cert,
|
||||||
|
verify=self.verify_server)
|
||||||
|
if response.ok:
|
||||||
|
return response.json().get('status')
|
||||||
|
raise exc.K8sClientException(response.text)
|
||||||
|
|
||||||
def post(self, path, body):
|
def post(self, path, body):
|
||||||
LOG.debug("Post %(path)s: %(body)s", {'path': path, 'body': body})
|
LOG.debug("Post %(path)s: %(body)s", {'path': path, 'body': body})
|
||||||
url = self._base_url + path
|
url = self._base_url + path
|
||||||
|
|
|
@ -214,6 +214,7 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
|
||||||
super(TestSriovDriver, self).setUp()
|
super(TestSriovDriver, self).setUp()
|
||||||
self.vif = fake._fake_vif(objects.vif.VIFSriov)
|
self.vif = fake._fake_vif(objects.vif.VIFSriov)
|
||||||
self.vif.physnet = 'test_physnet'
|
self.vif.physnet = 'test_physnet'
|
||||||
|
self.pci_info = mock.Mock()
|
||||||
|
|
||||||
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
|
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
|
||||||
'_get_host_pf_names')
|
'_get_host_pf_names')
|
||||||
|
@ -222,7 +223,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
|
||||||
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
|
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
|
||||||
'_set_vf_mac')
|
'_set_vf_mac')
|
||||||
def test_connect(self, m_set_vf_mac, m_avail_vf_info, m_host_pf_names):
|
def test_connect(self, m_set_vf_mac, m_avail_vf_info, m_host_pf_names):
|
||||||
m_avail_vf_info.return_value = [self.ifname, 1, 'h_interface']
|
m_avail_vf_info.return_value = [self.ifname, 1,
|
||||||
|
'h_interface', self.pci_info]
|
||||||
m_host_pf_names.return_value = 'h_interface'
|
m_host_pf_names.return_value = 'h_interface'
|
||||||
self._test_connect()
|
self._test_connect()
|
||||||
|
|
||||||
|
@ -232,5 +234,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
|
||||||
m_set_vf_mac.assert_called_once_with('h_interface', 1,
|
m_set_vf_mac.assert_called_once_with('h_interface', 1,
|
||||||
str(self.vif.address))
|
str(self.vif.address))
|
||||||
|
|
||||||
def test_disconnect(self):
|
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
|
||||||
|
'_remove_pci_info')
|
||||||
|
def test_disconnect(self, m_remove_pci):
|
||||||
|
m_remove_pci.return_value = None
|
||||||
self._test_disconnect()
|
self._test_disconnect()
|
||||||
|
|
Loading…
Reference in New Issue