Annotate nodes with pci info for direct ports

This commit allows to add pci information for direct
neutron ports attached to PODs into nodes annotations.
It happens on binding stage when pci information can
be requested.

Also this commit allows to delete annotations for such
ports when apropriate POD is deleted and VFs are returned
into host's network namespace.

For future commits: it is necessary to update neutron
ports with pci info when POD is in Running state.

Change-Id: I5ea8da6bb3143fd689e701f5e503488d4a6c9b33
Closes-Bug: 1818606
Signed-off-by: Danil Golov <d.golov@samsung.com>

kuryr_kubernetes/cni/binding/sriov.py

@@ -21,8 +21,10 @@ from oslo_concurrency import processutils
 from oslo_config import cfg
 from oslo_log import log as logging
 
+from kuryr_kubernetes import clients
 from kuryr_kubernetes.cni.binding import base as b_base
 from kuryr_kubernetes import config
+from kuryr_kubernetes import constants
 from kuryr_kubernetes import exceptions
 from kuryr_kubernetes import utils
 
@@ -53,7 +55,7 @@ class VIFSriovDriver(object):
         c_ipdb = b_base.get_ipdb(netns)
 
         pf_names = self._get_host_pf_names(physnet)
-        vf_name, vf_index, pf = self._get_available_vf_info(pf_names)
+        vf_name, vf_index, pf, pci_info = self._get_available_vf_info(pf_names)
 
         if not vf_name:
             error_msg = "No free interfaces for physnet {} available".format(
@@ -75,11 +77,13 @@ class VIFSriovDriver(object):
             iface.mtu = vif.network.mtu
             iface.up()
 
+        self._save_pci_info(vif.id, pci_info)
+
     def disconnect(self, vif, ifname, netns, container_id):
         # NOTE(k.zaitsev): when netns is deleted the interface is
         # returned automatically to host netns. We may reset
         # it to all-zero state
-        pass
+        self._remove_pci_info(vif.id)
 
     def _get_host_pf_names(self, physnet):
         """Return a list of PFs, that belong to a physnet"""
@@ -120,9 +124,70 @@ class VIFSriovDriver(object):
                     self._release()
                     continue
                 vf_name = vf_names[0]
+                pci_info = self._get_pci_info(pf, vf_index)
                 LOG.debug("Aquiring vf %s of pf %s", vf_index, pf)
-                return vf_name, vf_index, pf
+                return vf_name, vf_index, pf, pci_info
-        return None, None, None
+        return None, None, None, None
+
+    def _get_pci_info(self, pf, vf_index):
+        pci_slot = ''
+        physnet = ''
+        pci_vendor_info = ''
+
+        vendor_path = '/sys/class/net/{}/device/virtfn{}/vendor'.format(
+            pf, vf_index)
+        with open(vendor_path) as vendor_file:
+            vendor_full = vendor_file.read()
+            vendor = vendor_full.split('x')[1].strip()
+        device_path = '/sys/class/net/{}/device/virtfn{}/device'.format(
+            pf, vf_index)
+        with open(device_path) as device_file:
+            device_full = device_file.read()
+            device = device_full.split('x')[1].strip()
+        pci_vendor_info = '{}:{}'.format(vendor, device)
+
+        vf_path = '/sys/class/net/{}/device/virtfn{}'.format(
+            pf, vf_index)
+        pci_slot_path = os.readlink(vf_path)
+        pci_slot = pci_slot_path.split('/')[1]
+
+        physnet = self._get_physnet_by_pf(pf)
+
+        return {'pci_slot': pci_slot,
+                'physical_network': physnet,
+                'pci_vendor_info': pci_vendor_info}
+
+    def _get_physnet_by_pf(self, desired_pf):
+        for physnet, pf_list in self._device_pf_mapping.items():
+            for pf in pf_list:
+                if pf == desired_pf:
+                    return physnet
+        LOG.exception("Unable to find physnet for pf %s", desired_pf)
+        raise
+
+    def _save_pci_info(self, neutron_port, port_pci_info):
+        k8s = clients.get_kubernetes_client()
+        annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO
+        annot_name = annot_name.replace('/', '~1')
+        annot_name = annot_name + '-' + neutron_port
+        LOG.info('annot_name = %s', annot_name)
+        nodename = utils.get_node_name()
+
+        LOG.info("Trying to annotate node %s with pci info %s",
+                 nodename, port_pci_info)
+        k8s.patch_node_annotations(nodename, annot_name, port_pci_info)
+
+    def _remove_pci_info(self, neutron_port):
+        k8s = clients.get_kubernetes_client()
+        annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO
+        annot_name = annot_name.replace('/', '~1')
+        annot_name = annot_name + '-' + neutron_port
+        LOG.info('annot_name = %s', annot_name)
+        nodename = utils.get_node_name()
+
+        LOG.info("Trying to delete pci info for port %s on node %s",
+                 neutron_port, nodename)
+        k8s.remove_node_annotations(nodename, annot_name)
 
     def _acquire(self, path):
         if self._lock and self._lock.acquired:

kuryr_kubernetes/constants.py

@@ -52,6 +52,8 @@ K8S_ANNOTATION_NPWG_NETWORK = K8S_ANNOTATION_NPWG_PREFIX + '/networks'
 K8S_ANNOTATION_NPWG_CRD_SUBNET_ID = 'subnetId'
 K8S_ANNOTATION_NPWG_CRD_DRIVER_TYPE = 'driverType'
 
+K8S_ANNOTATION_NODE_PCI_DEVICE_INFO = 'openstack.org/kuryr-pci-info'
+
 K8S_OS_VIF_NOOP_PLUGIN = "noop"
 
 CNI_EXCEPTION_CODE = 100

kuryr_kubernetes/k8s_client.py

@@ -22,6 +22,7 @@ import requests
 
 from kuryr.lib._i18n import _
 from kuryr_kubernetes import config
+from kuryr_kubernetes import constants
 from kuryr_kubernetes import exceptions as exc
 
 LOG = logging.getLogger(__name__)
@@ -125,6 +126,38 @@ class K8sClient(object):
             return response.json().get('status')
         raise exc.K8sClientException(response.text)
 
+    def patch_node_annotations(self, node, annotation_name, value):
+        content_type = 'application/json-patch+json'
+        path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node)
+        value = jsonutils.dumps(value)
+        url, header = self._get_url_and_header(path, content_type)
+
+        data = [{'op': 'add',
+                 'path': '/metadata/annotations/{}'.format(annotation_name),
+                 'value': value}]
+
+        response = requests.patch(url, data=jsonutils.dumps(data),
+                                  headers=header, cert=self.cert,
+                                  verify=self.verify_server)
+        if response.ok:
+            return response.json().get('status')
+        raise exc.K8sClientException(response.text)
+
+    def remove_node_annotations(self, node, annotation_name):
+        content_type = 'application/json-patch+json'
+        path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node)
+        url, header = self._get_url_and_header(path, content_type)
+
+        data = [{'op': 'remove',
+                 'path': '/metadata/annotations/{}'.format(annotation_name)}]
+
+        response = requests.patch(url, data=jsonutils.dumps(data),
+                                  headers=header, cert=self.cert,
+                                  verify=self.verify_server)
+        if response.ok:
+            return response.json().get('status')
+        raise exc.K8sClientException(response.text)
+
     def post(self, path, body):
         LOG.debug("Post %(path)s: %(body)s", {'path': path, 'body': body})
         url = self._base_url + path

kuryr_kubernetes/tests/unit/cni/test_binding.py

@@ -214,6 +214,7 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
         super(TestSriovDriver, self).setUp()
         self.vif = fake._fake_vif(objects.vif.VIFSriov)
         self.vif.physnet = 'test_physnet'
+        self.pci_info = mock.Mock()
 
     @mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
                 '_get_host_pf_names')
@@ -222,7 +223,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
     @mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
                 '_set_vf_mac')
     def test_connect(self, m_set_vf_mac, m_avail_vf_info, m_host_pf_names):
-        m_avail_vf_info.return_value = [self.ifname, 1, 'h_interface']
+        m_avail_vf_info.return_value = [self.ifname, 1,
+                                        'h_interface', self.pci_info]
         m_host_pf_names.return_value = 'h_interface'
         self._test_connect()
 
@@ -232,5 +234,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase):
         m_set_vf_mac.assert_called_once_with('h_interface', 1,
                                              str(self.vif.address))
 
-    def test_disconnect(self):
+    @mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.'
+                '_remove_pci_info')
+    def test_disconnect(self, m_remove_pci):
+        m_remove_pci.return_value = None
         self._test_disconnect()