openstack
/
kuryr-kubernetes
Code Issues Proposed changes

Take care about OS resources which was not found on CRD. 

Neutron networks and subnets might be omitted during removing resources
on deletion of KuryrNetwork CRD. This might happen, when there is event
for resources creation, and some other calls might make Kuryr Controller
to crash, resulting in not tagged and/or missing netId field in KN.
Although resources are created while we don't have updated CRD, with the
naming convention which ensure its uniqueness we may now get the
potential leftovers to clean it properly.

Change-Id: I5e54d2ba08325935e4ee1da95372b34e0e40e0f0
This commit is contained in:
Roman Dobosz 2022-03-29 13:18:39 +02:00
parent 1bf9d6286c
commit d49add94df
2 changed files with 118 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
kuryr_kubernetes/controller/handlers/kuryrnetwork.py
View File

@ -139,6 +139,35 @@ class KuryrNetworkHandler(k8s_base.ResourceEventHandler):
# ports associated to the namespace/subnet, ensuring next
# retry will be successful
raise
else:
# NOTE(gryf): It may happen, that even if KuryrNetworkCRD was not
# updated (when controller crash in between), but the network and
# possibly subnet is there, so it needs to be searched manually.
ns = self.k8s.get(f'{constants.K8S_API_NAMESPACES}/'
f'{kuryrnet_crd["spec"]["nsName"]}')
ns_name = ns['metadata']['name']
ns_uid = ns['metadata']['uid']
net_name = driver_utils.get_resource_name(ns_name)
old_net_name = driver_utils.get_resource_name(ns_name,
prefix='ns/',
suffix='-net')
# TODO(gryf): remove old_net_name support in next release.
os_net = clients.get_network_client()
networks = os_net.networks(name=(net_name, old_net_name))
for net in networks:
if ns_uid == net.description:
LOG.warning('Found Neutron network associated with '
'namespace `%s`, while it is not registered '
'on KuryrNetwork CRD. Trying to remove.',
ns_name)
self._drv_vif_pool.delete_network_pools(net.id)
try:
os_net.delete_network(net)
except os_exc.ConflictException:
LOG.warning("One or more ports in use on the network "
"%s. Retrying.", net.id)
raise k_exc.ResourceNotReady(net.id)
namespace = {
'metadata': {'name': kuryrnet_crd['spec']['nsName']}}

90
kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrnetwork.py
View File

@ -14,6 +14,7 @@
from unittest import mock
import munch
from oslo_config import cfg as oslo_cfg
from kuryr_kubernetes.controller.drivers import base as drivers
@ -218,11 +219,18 @@ class TestKuryrNetworkHandler(test_base.TestCase):
self._handler.k8s.remove_finalizer.assert_called_once()
self._handler.k8s.add_event.assert_not_called()
@mock.patch('kuryr_kubernetes.clients.get_network_client')
@mock.patch.object(driver_utils, 'get_services')
def test_on_finalize_no_network(self, m_get_svc):
def test_on_finalize_no_network(self, m_get_svc, m_get_net_client):
crd_selector = mock.sentinel.crd_selector
self._delete_ns_sg_rules.return_value = [crd_selector]
m_get_svc.return_value = []
net_mock = mock.MagicMock()
net_mock.return_value = []
m_get_net_client.return_value = net_mock
self._handler.k8s.get.return_value = (
{'metadata': {'name': 'test-namespace',
'uid': 'e237acaf-ea5f-4d96-b604-292268a938a1'}})
kuryrnetwork.KuryrNetworkHandler.on_finalize(self._handler,
self._kuryrnet_crd)
@ -235,6 +243,86 @@ class TestKuryrNetworkHandler(test_base.TestCase):
self._handler.k8s.remove_finalizer.assert_called_once()
self._handler.k8s.add_event.assert_not_called()
@mock.patch('kuryr_kubernetes.clients.get_network_client')
@mock.patch.object(driver_utils, 'get_services')
def test_on_finalize_no_network_in_kn_no_desc(self, m_get_svc,
m_get_net_client):
crd_selector = mock.sentinel.crd_selector
self._delete_ns_sg_rules.return_value = [crd_selector]
m_get_svc.return_value = []
net_mock = mock.MagicMock()
net_id = '1612ffb1-ff7d-4590-bd9c-95aeb043705a'
net_mock.networks.return_value = [munch.Munch({'id': net_id,
'description': ''})]
m_get_net_client.return_value = net_mock
self._handler.k8s.get.return_value = (
{'metadata': {'name': 'test-namespace', 'uid': net_id}})
kuryrnetwork.KuryrNetworkHandler.on_finalize(self._handler,
self._kuryrnet_crd)
self._delete_network_pools.assert_not_called()
self._delete_ns_sg_rules.assert_called_once()
m_get_svc.assert_called_once()
self._handler._update_services.assert_called_once()
self._handler.k8s.remove_finalizer.assert_called_once()
self._handler.k8s.add_event.assert_not_called()
@mock.patch('kuryr_kubernetes.clients.get_network_client')
@mock.patch.object(driver_utils, 'get_services')
def test_on_finalize_no_network_in_kn_with_subnet(self, m_get_svc,
m_get_net_client):
crd_selector = mock.sentinel.crd_selector
self._delete_ns_sg_rules.return_value = [crd_selector]
m_get_svc.return_value = []
net_id = 'db0b2c83-dae3-47ff-b4b0-9a19b7c15589'
ns_id = '0b6d6f0b-4e44-4a1b-a711-71ab6c79bee8'
subnet_id = 'a595fc4b-6885-48ff-b90c-d3f7aefd6d1a'
net_mock = mock.MagicMock()
net_mock.networks.return_value = [munch.Munch({'id': net_id,
'description': ns_id})]
net_mock.subnets.return_value = [munch.Munch({'id': subnet_id})]
m_get_net_client.return_value = net_mock
self._handler.k8s.get.return_value = (
{'metadata': {'name': 'test-namespace', 'uid': ns_id}})
kuryrnetwork.KuryrNetworkHandler.on_finalize(self._handler,
self._kuryrnet_crd)
self._delete_network_pools.assert_called_once()
self._delete_ns_sg_rules.assert_called_once()
m_get_svc.assert_called_once()
self._handler._update_services.assert_called_once()
self._handler.k8s.remove_finalizer.assert_called_once()
self._handler.k8s.add_event.assert_not_called()
@mock.patch('kuryr_kubernetes.clients.get_network_client')
@mock.patch.object(driver_utils, 'get_services')
def test_on_finalize_no_network_in_kn_with_no_ns_match(self, m_get_svc,
m_get_net_client):
crd_selector = mock.sentinel.crd_selector
self._delete_ns_sg_rules.return_value = [crd_selector]
m_get_svc.return_value = []
net_id = 'db0b2c83-dae3-47ff-b4b0-9a19b7c15589'
ns_id = '0b6d6f0b-4e44-4a1b-a711-71ab6c79bee8'
subnet_id = 'a595fc4b-6885-48ff-b90c-d3f7aefd6d1a'
net_mock = mock.MagicMock()
net_mock.networks.return_value = [munch.Munch({'id': net_id,
'description': ns_id})]
net_mock.subnets.return_value = [munch.Munch({'id': subnet_id})]
m_get_net_client.return_value = net_mock
self._handler.k8s.get.return_value = (
{'metadata': {'name': 'test-namespace', 'uid': net_id}})
kuryrnetwork.KuryrNetworkHandler.on_finalize(self._handler,
self._kuryrnet_crd)
self._delete_network_pools.assert_not_called()
m_get_svc.assert_called_once()
self._handler._update_services.assert_called_once()
self._handler.k8s.remove_finalizer.assert_called_once()
self._handler.k8s.add_event.assert_not_called()
@mock.patch.object(driver_utils, 'get_services')
def test_on_finalize_no_sg_enforce(self, m_get_svc):
net_id = mock.sentinel.net_id