[WIP] Allow setting specific ports for SRIOV handler
Added possibility for SRIOV handler to parse pod annotations and to get specific ports. If it is not specified then adresses are requested from Neutron in specified subnet commonly. The total amount of requested ports is equal to amount of requested SRIOV VFs. Specific ports specified in pod annotations are not deleted after deployment deleting. Targets bp: kuryr-kubernetes-sriov-support Change-Id: I733199fcbf9298d4183e419538ac17f206797cde
This commit is contained in:
Danil Golov
parent
80d5f28ea2
commit
ad17796c31
|
|
@ -47,13 +47,17 @@ class SRIOVVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
|||
|
||||
ALIAS = 'sriov_pod_vif'
|
||||
|
||||
def request_vif(self, pod, project_id, physnets, security_groups):
|
||||
def request_vif(self, pod, project_id, physnets, security_groups,
|
||||
specific_ports):
|
||||
|
||||
sriov_requests = self._get_sriov_requests(pod)
|
||||
if not sriov_requests:
|
||||
return []
|
||||
|
||||
neutron = clients.get_neutron_client()
|
||||
spec_ports_by_pnets = self._get_spec_ports_pnets(specific_ports,
|
||||
neutron)
|
||||
LOG.debug("Specific ports by physnets {}".format(spec_ports_by_pnets))
|
||||
|
||||
pod_name = pod['metadata']['name']
|
||||
|
||||
|
|
@ -63,16 +67,35 @@ class SRIOVVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
|||
|
||||
subnets = physnets[physnet]
|
||||
|
||||
pnet_spec_ports = spec_ports_by_pnets.get(physnet, [])
|
||||
num_spec_ports = min(amount, len(pnet_spec_ports))
|
||||
|
||||
vif_plugin = 'sriov'
|
||||
for i in range(num_spec_ports):
|
||||
port = pnet_spec_ports[i]
|
||||
vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets)
|
||||
vif.physnet = physnet
|
||||
vifs.append(vif)
|
||||
|
||||
LOG.debug("Retrived vifs with specific ports {}".format(vifs))
|
||||
|
||||
num_retrived_spec_vifs = len(vifs)
|
||||
|
||||
if num_retrived_spec_vifs < len(pnet_spec_ports):
|
||||
LOG.warning("Not all specific ports were used.")
|
||||
|
||||
rq = self._get_port_request(
|
||||
pod, project_id, subnets, security_groups)
|
||||
|
||||
vif_plugin = 'sriov'
|
||||
for _ in range(amount):
|
||||
for _ in range(amount - num_retrived_spec_vifs):
|
||||
port = neutron.create_port(rq).get('port')
|
||||
vif = ovu.neutron_to_osvif_vif(vif_plugin, port, subnets)
|
||||
vif.physnet = physnet
|
||||
vifs.append(vif)
|
||||
|
||||
LOG.debug("All retrived vifs for physnet {}: {}".format(physnet,
|
||||
vifs))
|
||||
|
||||
LOG.debug("Got {} vifs for physnet {}, Pod {}".format(
|
||||
amount, physnet, pod_name))
|
||||
return vifs
|
||||
|
|
@ -114,3 +137,17 @@ class SRIOVVIFDriver(neutron_vif.NeutronPodVIFDriver):
|
|||
constants.K8S_OIR_SRIOV_PREFIX, '', 1)
|
||||
sriov_requests[physnet_name] += int(req_value)
|
||||
return sriov_requests
|
||||
|
||||
def _get_spec_ports_pnets(self, spec_ports, client):
|
||||
"""Function returns dict in format {physnet_name:[ports], ...}"""
|
||||
phys_ports = {}
|
||||
|
||||
for port in spec_ports:
|
||||
|
||||
network_id = port['network_id']
|
||||
network = client.show_network(network_id).get('network')
|
||||
physnet_id = network['provider:physical_network']
|
||||
|
||||
phys_ports.setdefault(physnet_id, []).append(port)
|
||||
|
||||
return phys_ports
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from kuryr_kubernetes.controller.drivers import base as drivers
|
|||
from kuryr_kubernetes.controller.drivers import sriov as sriov_drivers
|
||||
from kuryr_kubernetes import exceptions as k_exc
|
||||
from kuryr_kubernetes.handlers import k8s_base
|
||||
from kuryr_kubernetes import os_vif_util as ovu
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -66,6 +67,19 @@ class VIFHandler(k8s_base.ResourceEventHandler):
|
|||
vifs = self._get_vifs(pod)
|
||||
|
||||
if not vifs:
|
||||
|
||||
specific_port_ids = self._get_specific_port_ids(pod)
|
||||
LOG.debug("Specific port ids {} for pod".format(
|
||||
specific_port_ids))
|
||||
|
||||
neutron = clients.get_neutron_client()
|
||||
self._add_port_info(pod, specific_port_ids, neutron)
|
||||
|
||||
direct_ports, non_direct_ports = \
|
||||
self._get_objects_for_spec_port_ids(specific_port_ids)
|
||||
LOG.debug("Direct ports {}. Non-direct ports {}".format(
|
||||
direct_ports, non_direct_ports))
|
||||
|
||||
vifs = {}
|
||||
|
||||
project_id = self._drv_project.get_project(pod)
|
||||
|
|
@ -79,10 +93,16 @@ class VIFHandler(k8s_base.ResourceEventHandler):
|
|||
pod, project_id, subnets, security_groups)
|
||||
vifs[constants.DEFAULT_IFNAME] = main_vif
|
||||
|
||||
for port_id, port in enumerate(non_direct_ports):
|
||||
vif_translator = port.get('binding:vif_type')
|
||||
vif = ovu.neutron_to_osvif_vif(vif_translator, port, subnets)
|
||||
vifs['eth{}'.format(port_id + 1)] = vif
|
||||
|
||||
sriov_subnets = self._drv_sriov_subnets.get_subnets(pod,
|
||||
project_id)
|
||||
sriov_vifs = self._drv_sriov_vif.request_vif(
|
||||
pod, project_id, sriov_subnets, security_groups)
|
||||
pod, project_id, sriov_subnets, security_groups,
|
||||
direct_ports)
|
||||
|
||||
for vif_id, vif in enumerate(sriov_vifs):
|
||||
vifs['net{}'.format(vif_id)] = vif
|
||||
|
|
@ -111,10 +131,14 @@ class VIFHandler(k8s_base.ResourceEventHandler):
|
|||
project_id = self._drv_project.get_project(pod)
|
||||
security_groups = self._drv_sg.get_security_groups(pod, project_id)
|
||||
|
||||
persistent_ports = self._get_specific_port_ids(pod)
|
||||
LOG.debug("Persistent ports: %s", persistent_ports)
|
||||
|
||||
vifs = self._get_vifs(pod)
|
||||
for ifname, vif in vifs.items():
|
||||
self._drv_for_vif(vif).release_vif(pod, vif, project_id,
|
||||
security_groups)
|
||||
if vif.id not in persistent_ports:
|
||||
self._drv_for_vif(vif).release_vif(pod, vif, project_id,
|
||||
security_groups)
|
||||
|
||||
def _drv_for_vif(self, vif):
|
||||
# TODO(kzaitsev): a better polymorphism is required here
|
||||
|
|
@ -168,3 +192,43 @@ class VIFHandler(k8s_base.ResourceEventHandler):
|
|||
}
|
||||
LOG.debug("Got VIFs from annotation: %r", vifs)
|
||||
return vifs
|
||||
|
||||
def _get_specific_port_ids(self, pod):
|
||||
"""Function retunts list of ports not to be deleted in current pod"""
|
||||
ports = []
|
||||
try:
|
||||
annotations = pod['metadata']['annotations']
|
||||
except KeyError:
|
||||
return ports
|
||||
for annot_name, annot_value in annotations.items():
|
||||
if annot_name == 'spec-ports':
|
||||
ports += jsonutils.loads(annot_value)
|
||||
|
||||
return ports
|
||||
|
||||
def _get_objects_for_spec_port_ids(self, spec_port_ids):
|
||||
direct_ports = []
|
||||
non_direct_ports = []
|
||||
|
||||
neutron = clients.get_neutron_client()
|
||||
|
||||
for port_id in spec_port_ids:
|
||||
port = neutron.show_port(port_id).get('port')
|
||||
if port['binding:vnic_type'] == 'direct':
|
||||
direct_ports.append(port)
|
||||
else:
|
||||
non_direct_ports.append(port)
|
||||
|
||||
return direct_ports, non_direct_ports
|
||||
|
||||
def _add_port_info(self, pod, port_ids, client):
|
||||
for port_id in port_ids:
|
||||
client.update_port(
|
||||
port_id,
|
||||
{
|
||||
"port": {
|
||||
'device_owner': 'compute:kuryr',
|
||||
'device_id': pod['metadata']['uid'],
|
||||
'binding:host_id': pod['spec']['nodeName']
|
||||
}
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,144 @@
|
|||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import mock
|
||||
|
||||
from kuryr_kubernetes import constants as k_const
|
||||
from kuryr_kubernetes.controller.drivers import sriov as sriov_drivers
|
||||
from kuryr_kubernetes import os_vif_util as ovu
|
||||
from kuryr_kubernetes.tests import base as test_base
|
||||
from kuryr_kubernetes.tests.unit import kuryr_fixtures as k_fix
|
||||
|
||||
|
||||
class TestSRIOVVIFDriver(test_base.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSRIOVVIFDriver, self).setUp()
|
||||
|
||||
self._pod_version = mock.sentinel.pod_version
|
||||
self._pod_link = mock.sentinel.pod_link
|
||||
|
||||
self._subnet = mock.Mock()
|
||||
self._physnets = {
|
||||
'phy': {
|
||||
'sub_id': self._subnet
|
||||
}
|
||||
}
|
||||
|
||||
self._pod = {
|
||||
'metadata': {'resourceVersion': self._pod_version,
|
||||
'selfLink': self._pod_link,
|
||||
'name': 'podname'},
|
||||
'status': {'phase': k_const.K8S_POD_STATUS_PENDING},
|
||||
'spec': {
|
||||
'hostNetwork': False,
|
||||
'nodeName': 'hostname',
|
||||
'containers': [{
|
||||
'resources': {
|
||||
'requests': {
|
||||
'pod.alpha.kubernetes.io/'
|
||||
'opaque-int-resource-sriov-vf-physnet4': "3"
|
||||
}
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
|
||||
self._handler = mock.MagicMock(spec=sriov_drivers.SRIOVVIFDriver)
|
||||
|
||||
def test_get_spec_ports_pnets(self):
|
||||
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||
|
||||
physnet_id = 'id'
|
||||
network = {'provider:physical_network': physnet_id}
|
||||
port = {'network_id': network}
|
||||
|
||||
neutron.show_network.return_value = {'network': network}
|
||||
|
||||
cls = sriov_drivers.SRIOVVIFDriver()
|
||||
|
||||
port_ids = [port]
|
||||
expected_res = {'id': [port]}
|
||||
|
||||
self.assertEqual(expected_res,
|
||||
cls._get_spec_ports_pnets(port_ids,
|
||||
neutron))
|
||||
|
||||
def test_get_sriov_requests(self):
|
||||
cls = sriov_drivers.SRIOVVIFDriver()
|
||||
self.assertEqual({'physnet4': 3},
|
||||
cls._get_sriov_requests(self._pod))
|
||||
|
||||
def test_get_sriov_requests_no_req(self):
|
||||
self._pod['spec']['containers'] = [{}]
|
||||
cls = sriov_drivers.SRIOVVIFDriver()
|
||||
self.assertEqual({}, cls._get_sriov_requests(self._pod))
|
||||
|
||||
@mock.patch.object(ovu, 'neutron_to_osvif_vif')
|
||||
def test_request_vif(self, n_to_osvif_vif):
|
||||
project_id = mock.sentinel.project_id
|
||||
security_groups = mock.sentinel.security_groups
|
||||
specific_ports = mock.sentinel.specific_ports
|
||||
port1 = mock.sentinel.port1
|
||||
port2 = mock.sentinel.port2
|
||||
specific_ports = [port1, port2]
|
||||
|
||||
self._get_sriov_requests = self._handler._get_sriov_requests
|
||||
self._get_sriov_requests.return_value = {'phy': 3}
|
||||
|
||||
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||
|
||||
self._get_spec_ports_pnets = self._handler._get_spec_ports_pnets
|
||||
self._get_spec_ports_pnets.return_value = {'phy': [port1, port2]}
|
||||
|
||||
self._get_port_request = self._handler._get_port_request
|
||||
self._get_port_request.return_value = 'request'
|
||||
|
||||
vif = mock.Mock()
|
||||
n_to_osvif_vif.return_value = vif
|
||||
|
||||
cls = sriov_drivers.SRIOVVIFDriver
|
||||
cls.request_vif(self._handler, self._pod, project_id,
|
||||
self._physnets, security_groups,
|
||||
specific_ports)
|
||||
|
||||
self._get_sriov_requests.assert_called_once_with(self._pod)
|
||||
|
||||
expected_calls_create = [mock.call('request')]
|
||||
|
||||
neutron.create_port.assert_has_calls(expected_calls_create,
|
||||
any_order=True)
|
||||
|
||||
def test_request_vif_no_requests(self):
|
||||
project_id = mock.sentinel.project_id
|
||||
security_groups = mock.sentinel.security_groups
|
||||
specific_ports = mock.sentinel.specific_ports
|
||||
|
||||
self._get_sriov_requests = self._handler._get_sriov_requests
|
||||
self._get_sriov_requests.return_value = {}
|
||||
|
||||
neutron = self.useFixture(k_fix.MockNeutronClient()).client
|
||||
|
||||
self._get_spec_ports_pnets = self._handler._get_spec_ports_pnets
|
||||
self._get_spec_ports_pnets.return_value = mock.sentinel.spec_ports_phy
|
||||
|
||||
cls = sriov_drivers.SRIOVVIFDriver
|
||||
cls.request_vif(self._handler, self._pod, project_id,
|
||||
self._physnets, security_groups,
|
||||
specific_ports)
|
||||
|
||||
self._get_sriov_requests.assert_called_once_with(self._pod)
|
||||
|
||||
neutron.show_port.assert_not_called()
|
||||
neutron.create_port.assert_not_called()
|
||||
|
|
@ -15,11 +15,13 @@
|
|||
|
||||
import mock
|
||||
|
||||
from kuryr_kubernetes import clients
|
||||
from kuryr_kubernetes import constants as k_const
|
||||
from kuryr_kubernetes.controller.drivers import base as drivers
|
||||
from kuryr_kubernetes.controller.drivers import sriov as sriov_drivers
|
||||
from kuryr_kubernetes.controller.handlers import vif as h_vif
|
||||
from kuryr_kubernetes import exceptions as k_exc
|
||||
from kuryr_kubernetes import os_vif_util as ovu
|
||||
from kuryr_kubernetes.tests import base as test_base
|
||||
|
||||
|
||||
|
|
@ -84,6 +86,8 @@ class TestVIFHandler(test_base.TestCase):
|
|||
self._sriov_request_vif.return_value = {}
|
||||
self._handler._drv_for_vif = h_vif.VIFHandler._drv_for_vif.__get__(
|
||||
self._handler)
|
||||
self._get_objects_for_spec_port_ids = (
|
||||
self._handler._get_objects_for_spec_port_ids)
|
||||
|
||||
@mock.patch.object(drivers.VIFPoolDriver, 'set_vif_driver')
|
||||
@mock.patch.object(drivers.VIFPoolDriver, 'get_instance')
|
||||
|
|
@ -177,36 +181,71 @@ class TestVIFHandler(test_base.TestCase):
|
|||
self._set_vifs.assert_called_once_with(self._pod, self._vifs)
|
||||
self._request_vif.assert_not_called()
|
||||
|
||||
def test_on_present_create(self):
|
||||
@mock.patch.object(ovu, 'neutron_to_osvif_vif')
|
||||
@mock.patch.object(clients, 'get_neutron_client')
|
||||
def test_on_present_create(self, get_neutron_client, n_to_osvif_vif):
|
||||
self._get_vifs.return_value = {}
|
||||
get_neutron_client.return_value = mock.sentinel.neutron
|
||||
port = mock.Mock()
|
||||
self._get_objects_for_spec_port_ids.return_value = ([port], [port])
|
||||
vif = mock.Mock()
|
||||
n_to_osvif_vif.return_value = vif
|
||||
|
||||
d1 = self._vifs.copy()
|
||||
d2 = dict([('eth1', vif)])
|
||||
d1.update(d2)
|
||||
|
||||
h_vif.VIFHandler.on_present(self._handler, self._pod)
|
||||
|
||||
self._get_vifs.assert_called_once_with(self._pod)
|
||||
self._request_vif.assert_called_once_with(
|
||||
self._pod, self._project_id, self._subnets, self._security_groups)
|
||||
self._set_vifs.assert_called_once_with(self._pod, self._vifs)
|
||||
self._set_vifs.assert_called_once_with(self._pod, d1)
|
||||
self._activate_vif.assert_not_called()
|
||||
|
||||
def test_on_present_rollback(self):
|
||||
@mock.patch.object(ovu, 'neutron_to_osvif_vif')
|
||||
@mock.patch.object(clients, 'get_neutron_client')
|
||||
def test_on_present_rollback(self, get_neutron_client, n_to_osvif_vif):
|
||||
self._get_vifs.return_value = {}
|
||||
self._set_vifs.side_effect = k_exc.K8sClientException
|
||||
get_neutron_client.return_value = mock.sentinel.neutron
|
||||
port = mock.Mock()
|
||||
self._get_objects_for_spec_port_ids.return_value = ([port], [port])
|
||||
vif = mock.Mock()
|
||||
n_to_osvif_vif.return_value = vif
|
||||
|
||||
d1 = self._vifs.copy()
|
||||
d2 = dict([('eth1', vif)])
|
||||
d1.update(d2)
|
||||
|
||||
h_vif.VIFHandler.on_present(self._handler, self._pod)
|
||||
|
||||
self._get_vifs.assert_called_once_with(self._pod)
|
||||
self._request_vif.assert_called_once_with(
|
||||
self._pod, self._project_id, self._subnets, self._security_groups)
|
||||
self._set_vifs.assert_called_once_with(self._pod, self._vifs)
|
||||
self._release_vif.assert_called_once_with(self._pod, self._vif,
|
||||
self._project_id,
|
||||
self._security_groups)
|
||||
self._set_vifs.assert_called_once_with(self._pod, d1)
|
||||
|
||||
expected_calls = [mock.call(self._pod, self._vif, self._project_id,
|
||||
self._security_groups),
|
||||
mock.call(self._pod, vif, self._project_id,
|
||||
self._security_groups)]
|
||||
|
||||
self._release_vif.assert_has_calls(expected_calls, any_order=True)
|
||||
self._activate_vif.assert_not_called()
|
||||
|
||||
def test_on_deleted(self):
|
||||
self._get_vifs.return_value = {'physnet': self._vif}
|
||||
self._get_specific_port_ids = self._handler._get_specific_port_ids
|
||||
self._get_specific_port_ids.return_value = ['port1', 'port2']
|
||||
self._vif.id = 'port3'
|
||||
|
||||
h_vif.VIFHandler.on_deleted(self._handler, self._pod)
|
||||
|
||||
self._get_project.assert_called_once_with(self._pod)
|
||||
self._get_security_groups.assert_called_once_with(self._pod,
|
||||
self._project_id)
|
||||
self._get_vifs.assert_called_once_with(self._pod)
|
||||
self._get_specific_port_ids.assert_called_once_with(self._pod)
|
||||
self._release_vif.assert_called_once_with(self._pod, self._vif,
|
||||
self._project_id,
|
||||
self._security_groups)
|
||||
|
|
@ -226,3 +265,15 @@ class TestVIFHandler(test_base.TestCase):
|
|||
|
||||
self._get_vifs.assert_called_once_with(self._pod)
|
||||
self._release_vif.assert_not_called()
|
||||
|
||||
def test_get_specific_ports_no_annot(self):
|
||||
cls = h_vif.VIFHandler()
|
||||
self.assertEqual([], cls._get_specific_port_ids(self._pod))
|
||||
|
||||
def test_get_specific_ports(self):
|
||||
pod_spec_ports = {'spec-ports': '["address", "address"]'}
|
||||
self._pod['metadata']['annotations'] = pod_spec_ports
|
||||
cls = h_vif.VIFHandler()
|
||||
expected_ports = ['address', 'address']
|
||||
self.assertEqual(expected_ports,
|
||||
cls._get_specific_port_ids(self._pod))
|
||||
|
|
|
|||
Loading…
Reference in New Issue