Ensure namespace are recreated upon a k8s client failure

If k8s client is not responsive after the neutron resources for a
namespace are generated a rollback action will be triggered. However
if during that rollback action the k8s is still not accessible, only
the neutron resources will be deleted and the k8s ones will be left
behind, blocking the recreation of the needed neutron resources

Change-Id: I3500b11717f6eeeb5b5686a25f085d26900aa434
Closes-Bug: 1856000
(cherry picked from commit 64490e5d13)
This commit is contained in:
Luis Tomas Bolivar 2019-12-16 13:18:32 +01:00
parent cc166ae3d9
commit c23fd82a71
2 changed files with 43 additions and 4 deletions

View File

@ -125,8 +125,8 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
# create CRD resource for the network
try:
net_crd = self._add_kuryrnet_crd(ns_name, net_crd_spec)
self._set_net_crd(namespace, net_crd)
self._drv_sg.create_namespace_sg_rules(namespace)
self._set_net_crd(namespace, net_crd)
except (exceptions.K8sClientException,
exceptions.K8sResourceNotFound):
LOG.exception("Kuryrnet CRD creation failed. Rolling back "
@ -134,7 +134,12 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name)
if net_crd_sg.get('sgId'):
self._drv_sg.delete_sg(net_crd_sg['sgId'])
self._del_kuryrnet_crd(net_crd_name)
try:
self._del_kuryrnet_crd(net_crd_name)
except exceptions.K8sClientException:
LOG.exception("Error when trying to rollback the KuryrNet CRD "
"object %s", net_crd_name)
raise exceptions.ResourceNotReady(namespace)
def on_deleted(self, namespace, net_crd=None):
LOG.debug("Deleting namespace: %s", namespace)

View File

@ -205,7 +205,9 @@ class TestNamespaceHandler(test_base.TestCase):
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
self._add_kuryrnet_crd.side_effect = k_exc.K8sClientException
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
self.assertRaises(k_exc.ResourceNotReady,
namespace.NamespaceHandler.on_present,
self._handler, self._namespace)
self._get_net_crd_id.assert_called_once_with(self._namespace)
self._get_net_crd.assert_called_once_with(self._crd_id)
@ -231,7 +233,9 @@ class TestNamespaceHandler(test_base.TestCase):
self._add_kuryrnet_crd.return_value = net_crd
self._set_net_crd.side_effect = k_exc.K8sClientException
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
self.assertRaises(k_exc.ResourceNotReady,
namespace.NamespaceHandler.on_present,
self._handler, self._namespace)
self._get_net_crd_id.assert_called_once_with(self._namespace)
self._get_net_crd.assert_called_once_with(self._crd_id)
@ -278,6 +282,36 @@ class TestNamespaceHandler(test_base.TestCase):
self._set_net_crd.assert_called_once_with(self._namespace, net_crd)
self._rollback_network_resources.assert_called_once()
def test_on_present_del_kuryrnet_exception(self):
net_crd = self._get_crd()
self._get_net_crd_id.return_value = None
self._get_net_crd.return_value = None
self._create_namespace_network.return_value = {'test_net': 'uuid'}
self._create_namespace_sg.return_value = {'sgId': 'uuid'}
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
self._add_kuryrnet_crd.return_value = net_crd
self._set_net_crd.side_effect = k_exc.K8sClientException
self._del_kuryrnet_crd.side_effect = k_exc.K8sClientException
self.assertRaises(k_exc.ResourceNotReady,
namespace.NamespaceHandler.on_present,
self._handler, self._namespace)
self._get_net_crd_id.assert_called_once_with(self._namespace)
self._get_net_crd.assert_called_once_with(self._crd_id)
self._cleanup_namespace_networks.assert_called_once_with(
self._namespace_name)
self._create_namespace_network.assert_called_once_with(
self._namespace_name, self._project_id)
self._create_namespace_sg.assert_called_once_with(
self._namespace_name, self._project_id, net_crd_spec)
self._add_kuryrnet_crd.assert_called_once_with(self._namespace_name,
net_crd_spec)
self._set_net_crd.assert_called_once_with(self._namespace, net_crd)
self._rollback_network_resources.assert_called_once()
self._del_kuryrnet_crd.assert_called_once()
def test_on_deleted(self):
net_crd_id = mock.sentinel.net_crd_id
net_crd = self._get_crd()