From b0ce30142ececa9ad8d5508e9ed54930c63d3817 Mon Sep 17 00:00:00 2001 From: Danil Golov Date: Mon, 11 Mar 2019 15:15:26 +0300 Subject: [PATCH] Annotate nodes with pci info for direct ports This commit allows to add pci information for direct neutron ports attached to PODs into nodes annotations. It happens on binding stage when pci information can be requested. Also this commit allows to delete annotations for such ports when apropriate POD is deleted and VFs are returned into host's network namespace. For future commits: it is necessary to update neutron ports with pci info when POD is in Running state. Change-Id: I5ea8da6bb3143fd689e701f5e503488d4a6c9b33 Closes-Bug: 1818606 Signed-off-by: Danil Golov --- kuryr_kubernetes/cni/binding/sriov.py | 73 ++++++++++++++++++- kuryr_kubernetes/constants.py | 2 + kuryr_kubernetes/k8s_client.py | 33 +++++++++ .../tests/unit/cni/test_binding.py | 9 ++- 4 files changed, 111 insertions(+), 6 deletions(-) diff --git a/kuryr_kubernetes/cni/binding/sriov.py b/kuryr_kubernetes/cni/binding/sriov.py index 3b951c62b..20f00c967 100644 --- a/kuryr_kubernetes/cni/binding/sriov.py +++ b/kuryr_kubernetes/cni/binding/sriov.py @@ -21,8 +21,10 @@ from oslo_concurrency import processutils from oslo_config import cfg from oslo_log import log as logging +from kuryr_kubernetes import clients from kuryr_kubernetes.cni.binding import base as b_base from kuryr_kubernetes import config +from kuryr_kubernetes import constants from kuryr_kubernetes import exceptions from kuryr_kubernetes import utils @@ -53,7 +55,7 @@ class VIFSriovDriver(object): c_ipdb = b_base.get_ipdb(netns) pf_names = self._get_host_pf_names(physnet) - vf_name, vf_index, pf = self._get_available_vf_info(pf_names) + vf_name, vf_index, pf, pci_info = self._get_available_vf_info(pf_names) if not vf_name: error_msg = "No free interfaces for physnet {} available".format( @@ -75,11 +77,13 @@ class VIFSriovDriver(object): iface.mtu = vif.network.mtu iface.up() + self._save_pci_info(vif.id, pci_info) + def disconnect(self, vif, ifname, netns, container_id): # NOTE(k.zaitsev): when netns is deleted the interface is # returned automatically to host netns. We may reset # it to all-zero state - pass + self._remove_pci_info(vif.id) def _get_host_pf_names(self, physnet): """Return a list of PFs, that belong to a physnet""" @@ -120,9 +124,70 @@ class VIFSriovDriver(object): self._release() continue vf_name = vf_names[0] + pci_info = self._get_pci_info(pf, vf_index) LOG.debug("Aquiring vf %s of pf %s", vf_index, pf) - return vf_name, vf_index, pf - return None, None, None + return vf_name, vf_index, pf, pci_info + return None, None, None, None + + def _get_pci_info(self, pf, vf_index): + pci_slot = '' + physnet = '' + pci_vendor_info = '' + + vendor_path = '/sys/class/net/{}/device/virtfn{}/vendor'.format( + pf, vf_index) + with open(vendor_path) as vendor_file: + vendor_full = vendor_file.read() + vendor = vendor_full.split('x')[1].strip() + device_path = '/sys/class/net/{}/device/virtfn{}/device'.format( + pf, vf_index) + with open(device_path) as device_file: + device_full = device_file.read() + device = device_full.split('x')[1].strip() + pci_vendor_info = '{}:{}'.format(vendor, device) + + vf_path = '/sys/class/net/{}/device/virtfn{}'.format( + pf, vf_index) + pci_slot_path = os.readlink(vf_path) + pci_slot = pci_slot_path.split('/')[1] + + physnet = self._get_physnet_by_pf(pf) + + return {'pci_slot': pci_slot, + 'physical_network': physnet, + 'pci_vendor_info': pci_vendor_info} + + def _get_physnet_by_pf(self, desired_pf): + for physnet, pf_list in self._device_pf_mapping.items(): + for pf in pf_list: + if pf == desired_pf: + return physnet + LOG.exception("Unable to find physnet for pf %s", desired_pf) + raise + + def _save_pci_info(self, neutron_port, port_pci_info): + k8s = clients.get_kubernetes_client() + annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO + annot_name = annot_name.replace('/', '~1') + annot_name = annot_name + '-' + neutron_port + LOG.info('annot_name = %s', annot_name) + nodename = utils.get_node_name() + + LOG.info("Trying to annotate node %s with pci info %s", + nodename, port_pci_info) + k8s.patch_node_annotations(nodename, annot_name, port_pci_info) + + def _remove_pci_info(self, neutron_port): + k8s = clients.get_kubernetes_client() + annot_name = constants.K8S_ANNOTATION_NODE_PCI_DEVICE_INFO + annot_name = annot_name.replace('/', '~1') + annot_name = annot_name + '-' + neutron_port + LOG.info('annot_name = %s', annot_name) + nodename = utils.get_node_name() + + LOG.info("Trying to delete pci info for port %s on node %s", + neutron_port, nodename) + k8s.remove_node_annotations(nodename, annot_name) def _acquire(self, path): if self._lock and self._lock.acquired: diff --git a/kuryr_kubernetes/constants.py b/kuryr_kubernetes/constants.py index 68f638186..4a07dd8c6 100644 --- a/kuryr_kubernetes/constants.py +++ b/kuryr_kubernetes/constants.py @@ -52,6 +52,8 @@ K8S_ANNOTATION_NPWG_NETWORK = K8S_ANNOTATION_NPWG_PREFIX + '/networks' K8S_ANNOTATION_NPWG_CRD_SUBNET_ID = 'subnetId' K8S_ANNOTATION_NPWG_CRD_DRIVER_TYPE = 'driverType' +K8S_ANNOTATION_NODE_PCI_DEVICE_INFO = 'openstack.org/kuryr-pci-info' + K8S_OS_VIF_NOOP_PLUGIN = "noop" CNI_EXCEPTION_CODE = 100 diff --git a/kuryr_kubernetes/k8s_client.py b/kuryr_kubernetes/k8s_client.py index 8cdc7027d..16e4e7990 100644 --- a/kuryr_kubernetes/k8s_client.py +++ b/kuryr_kubernetes/k8s_client.py @@ -22,6 +22,7 @@ import requests from kuryr.lib._i18n import _ from kuryr_kubernetes import config +from kuryr_kubernetes import constants from kuryr_kubernetes import exceptions as exc LOG = logging.getLogger(__name__) @@ -125,6 +126,38 @@ class K8sClient(object): return response.json().get('status') raise exc.K8sClientException(response.text) + def patch_node_annotations(self, node, annotation_name, value): + content_type = 'application/json-patch+json' + path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node) + value = jsonutils.dumps(value) + url, header = self._get_url_and_header(path, content_type) + + data = [{'op': 'add', + 'path': '/metadata/annotations/{}'.format(annotation_name), + 'value': value}] + + response = requests.patch(url, data=jsonutils.dumps(data), + headers=header, cert=self.cert, + verify=self.verify_server) + if response.ok: + return response.json().get('status') + raise exc.K8sClientException(response.text) + + def remove_node_annotations(self, node, annotation_name): + content_type = 'application/json-patch+json' + path = '{}/nodes/{}/'.format(constants.K8S_API_BASE, node) + url, header = self._get_url_and_header(path, content_type) + + data = [{'op': 'remove', + 'path': '/metadata/annotations/{}'.format(annotation_name)}] + + response = requests.patch(url, data=jsonutils.dumps(data), + headers=header, cert=self.cert, + verify=self.verify_server) + if response.ok: + return response.json().get('status') + raise exc.K8sClientException(response.text) + def post(self, path, body): LOG.debug("Post %(path)s: %(body)s", {'path': path, 'body': body}) url = self._base_url + path diff --git a/kuryr_kubernetes/tests/unit/cni/test_binding.py b/kuryr_kubernetes/tests/unit/cni/test_binding.py index 1d14cae96..da96a160f 100644 --- a/kuryr_kubernetes/tests/unit/cni/test_binding.py +++ b/kuryr_kubernetes/tests/unit/cni/test_binding.py @@ -214,6 +214,7 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase): super(TestSriovDriver, self).setUp() self.vif = fake._fake_vif(objects.vif.VIFSriov) self.vif.physnet = 'test_physnet' + self.pci_info = mock.Mock() @mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.' '_get_host_pf_names') @@ -222,7 +223,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase): @mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.' '_set_vf_mac') def test_connect(self, m_set_vf_mac, m_avail_vf_info, m_host_pf_names): - m_avail_vf_info.return_value = [self.ifname, 1, 'h_interface'] + m_avail_vf_info.return_value = [self.ifname, 1, + 'h_interface', self.pci_info] m_host_pf_names.return_value = 'h_interface' self._test_connect() @@ -232,5 +234,8 @@ class TestSriovDriver(TestDriverMixin, test_base.TestCase): m_set_vf_mac.assert_called_once_with('h_interface', 1, str(self.vif.address)) - def test_disconnect(self): + @mock.patch('kuryr_kubernetes.cni.binding.sriov.VIFSriovDriver.' + '_remove_pci_info') + def test_disconnect(self, m_remove_pci): + m_remove_pci.return_value = None self._test_disconnect()