Annotate nodes with pci info for direct ports

This commit allows to add pci information for direct
neutron ports attached to PODs into nodes annotations.
It happens on binding stage when pci information can
be requested.

Also this commit allows to delete annotations for such
ports when apropriate POD is deleted and VFs are returned
into host's network namespace.

For future commits: it is necessary to update neutron
ports with pci info when POD is in Running state.

Change-Id: I5ea8da6bb3143fd689e701f5e503488d4a6c9b33
Closes-Bug: 1818606
Signed-off-by: Danil Golov <d.golov@samsung.com>
This commit is contained in:
Danil Golov 2019-03-11 15:15:26 +03:00
parent 3791b84069
commit b0ce30142e
4 changed files with 111 additions and 6 deletions

View File

@ -21,8 +21,10 @@ from oslo_concurrency import processutils
from oslo_config import cfg
from oslo_log import log as logging
from kuryr_kubernetes import clients
from kuryr_kubernetes.cni.binding import base as b_base
from kuryr_kubernetes import config
from kuryr_kubernetes import constants
from kuryr_kubernetes import exceptions
from kuryr_kubernetes import utils
@ -53,7 +55,7 @@ class VIFSriovDriver(object):
c_ipdb = b_base.get_ipdb(netns)
pf_names = self._get_host_pf_names(physnet)
vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
vf_name, vf_index, pf, pci_info = self._get_available_vf_info(pf_names)
if not vf_name:
error_msg = "No free interfaces for physnet {} available".format(
@ -75,11 +77,13 @@ class VIFSriovDriver(object):
iface.mtu = vif.network.mtu
iface.up()
self._save_pci_info(vif.id, pci_info)
def disconnect(self, vif, ifname, netns, container_id):
# NOTE(k.zaitsev): when netns is deleted the interface is
# returned automatically to host netns. We may reset
# it to all-zero state
pass
self._remove_pci_info(vif.id)
def _get_host_pf_names(self, physnet):
"""Return a list of PFs, that belong to a physnet"""
@ -120,9 +124,70 @@ class VIFSriovDriver(object):
self._release()
continue
vf_name = vf_names[0]
pci_info = self._get_pci_info(pf, vf_index)
LOG.debug("Aquiring vf %s of pf %s", vf_index, pf)
return vf_name, vf_index, pf
return None, None, None
return vf_name, vf_index, pf, pci_info
return None, None, None, None
def _get_pci_info(self, pf, vf_index):
pci_slot = ''
physnet = ''
pci_vendor_info = ''
vendor_path = '/sys/class/net/{}/device/virtfn{}/vendor'.format(
pf, vf_index)
with open(vendor_path) as vendor_file:
vendor_full = vendor_file.read()
vendor = vendor_full.split('x')[1].strip()
device_path = '/sys/class/net/{}/device/virtfn{}/device'.format(
pf, vf_index)
with open(device_path) as device_file:
device_full = device_file.read()
device = device_full.split('x')[1].strip()
pci_vendor_info = '{}:{}'.format(vendor, device)
vf_path = '/sys/class/net/{}/device/virtfn{}'.format(
pf, vf_index)
pci_slot_path = os.readlink(vf_path)
pci_slot = pci_slot_path.split('/')[1]
physnet = self._get_physnet_by_pf(pf)
return {'pci_slot': pci_slot,
'physical_network': physnet,
'pci_vendor_info': pci_vendor_info}
def _get_physnet_by_pf(self, desired_pf):
for physnet, pf_list in self._device_pf_mapping.items():
for pf in pf_list:
if pf == desired_pf:
return physnet
LOG.exception("Unable to find physnet for pf %s", desired_pf)
raise
def _save_pci_info(self, neutron_port, port_pci_info):
k8s = clients.get_kubernetes_client()
annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO
annot_name = annot_name.replace('/', '~1')
annot_name = annot_name + '-' + neutron_port
LOG.info('annot_name = %s', annot_name)
nodename = utils.get_node_name()
LOG.info("Trying to annotate node %s with pci info %s",
nodename, port_pci_info)
k8s.patch_node_annotations(nodename, annot_name, port_pci_info)
def _remove_pci_info(self, neutron_port):
k8s = clients.get_kubernetes_client()
annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO
annot_name = annot_name.replace('/', '~1')
annot_name = annot_name + '-' + neutron_port
LOG.info('annot_name = %s', annot_name)
nodename = utils.get_node_name()
LOG.info("Trying to delete pci info for port %s on node %s",
neutron_port, nodename)
k8s.remove_node_annotations(nodename, annot_name)
def _acquire(self, path):
if self._lock and self._lock.acquired:

View File

@ -52,6 +52,8 @@ K8S_ANNOTATION_NPWG_NETWORK = K8S_ANNOTATION_NPWG_PREFIX + '/networks'
K8S_ANNOTATION_NPWG_CRD_SUBNET_ID = 'subnetId'
K8S_ANNOTATION_NPWG_CRD_DRIVER_TYPE = 'driverType'
K8S_ANNOTATION_NODE_PCI_DEVICE_INFO = 'openstack.org/kuryr-pci-info'
K8S_OS_VIF_NOOP_PLUGIN = "noop"
CNI_EXCEPTION_CODE = 100

View File

@ -22,6 +22,7 @@ import requests
from kuryr.lib._i18n import _
from kuryr_kubernetes import config
from kuryr_kubernetes import constants
from kuryr_kubernetes import exceptions as exc
LOG = logging.getLogger(__name__)
@ -125,6 +126,38 @@ class K8sClient(object):
return response.json().get('status')
raise exc.K8sClientException(response.text)
def patch_node_annotations(self, node, annotation_name, value):
content_type = 'application/json-patch+json'
path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node)
value = jsonutils.dumps(value)
url, header = self._get_url_and_header(path, content_type)
data = [{'op': 'add',
'path': '/metadata/annotations/{}'.format(annotation_name),
'value': value}]
response = requests.patch(url, data=jsonutils.dumps(data),
headers=header, cert=self.cert,
verify=self.verify_server)
if response.ok:
return response.json().get('status')
raise exc.K8sClientException(response.text)
def remove_node_annotations(self, node, annotation_name):
content_type = 'application/json-patch+json'
path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node)
url, header = self._get_url_and_header(path, content_type)
data = [{'op': 'remove',
'path': '/metadata/annotations/{}'.format(annotation_name)}]
response = requests.patch(url, data=jsonutils.dumps(data),
headers=header, cert=self.cert,
verify=self.verify_server)
if response.ok:
return response.json().get('status')
raise exc.K8sClientException(response.text)
def post(self, path, body):
LOG.debug("Post %(path)s: %(body)s", {'path': path, 'body': body})
url = self._base_url + path

View File

@ -214,6 +214,7 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
super(TestSriovDriver, self).setUp()
self.vif = fake._fake_vif(objects.vif.VIFSriov)
self.vif.physnet = 'test_physnet'
self.pci_info = mock.Mock()
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
'_get_host_pf_names')
@ -222,7 +223,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
'_set_vf_mac')
def test_connect(self, m_set_vf_mac, m_avail_vf_info, m_host_pf_names):
m_avail_vf_info.return_value = [self.ifname, 1, 'h_interface']
m_avail_vf_info.return_value = [self.ifname, 1,
'h_interface', self.pci_info]
m_host_pf_names.return_value = 'h_interface'
self._test_connect()
@ -232,5 +234,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
m_set_vf_mac.assert_called_once_with('h_interface', 1,
str(self.vif.address))
def test_disconnect(self):
@mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
'_remove_pci_info')
def test_disconnect(self, m_remove_pci):
m_remove_pci.return_value = None
self._test_disconnect()