Fix fail to recreate namespace when previous KuryrNet CRD is not deleted
When a namespace/project is recreated right after its deletion,
it's possible that the previous kuryrNet CRD is not yet removed,
and Kuryr will try to create another CRD, resulting in k8s conflict
exception. In the meantime, other events handling resources under
the same namespace in question could be inquiring for an annotation
on the namespace, that was not annotated due to the conflict error,
causing the controller to restart.
This commit fixes the issue by adding an annotation to the KuryrNet
CRD with the respective namespace uid, which allows to check
if the existing CRD corresponds to the namespace.
Depends-On: https://review.opendev.org/#/c/677487/
Change-Id: I85d903902b516bfd109078705860540d731886bc
Closes-Bug: 1833712
(cherry picked from commit 37b8e3e6b3
)
This commit is contained in:
parent
a4104b35ac
commit
46b1407dea
|
@ -46,6 +46,15 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
LOG.debug("CRD existing at the new namespace")
|
||||
return
|
||||
|
||||
net_crd_name = 'ns-' + ns_name
|
||||
net_crd = self._get_net_crd(net_crd_name)
|
||||
if net_crd:
|
||||
LOG.debug("Previous CRD existing at the new namespace. "
|
||||
"Deleting namespace resources and retrying its "
|
||||
"creation.")
|
||||
self.on_deleted(namespace, net_crd)
|
||||
raise exceptions.ResourceNotReady(namespace)
|
||||
|
||||
LOG.debug("Creating network resources for namespace: %s", ns_name)
|
||||
net_crd_spec = self._drv_subnets.create_namespace_network(ns_name,
|
||||
project_id)
|
||||
|
@ -67,20 +76,29 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
try:
|
||||
net_crd = self._add_kuryrnet_crd(ns_name, net_crd_spec)
|
||||
self._set_net_crd(namespace, net_crd)
|
||||
except exceptions.K8sResourceNotFound:
|
||||
LOG.debug("Namespace could not be annotated. Rolling back "
|
||||
"resources created for it.")
|
||||
self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name)
|
||||
self._drv_sg.delete_sg(net_crd_sg['sgId'])
|
||||
self._del_kuryrnet_crd(net_crd_name)
|
||||
except exceptions.K8sClientException:
|
||||
LOG.exception("Kuryrnet CRD could not be added. Rolling back "
|
||||
"network resources created for the namespace.")
|
||||
self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name)
|
||||
self._drv_sg.delete_sg(net_crd_sg['sgId'])
|
||||
|
||||
def on_deleted(self, namespace):
|
||||
def on_deleted(self, namespace, net_crd=None):
|
||||
LOG.debug("Deleting namespace: %s", namespace)
|
||||
net_crd_id = self._get_net_crd_id(namespace)
|
||||
if not net_crd_id:
|
||||
LOG.warning("There is no CRD annotated at the namespace %s",
|
||||
namespace)
|
||||
return
|
||||
net_crd = self._get_net_crd(net_crd_id)
|
||||
if not net_crd:
|
||||
net_crd_id = self._get_net_crd_id(namespace)
|
||||
if not net_crd_id:
|
||||
LOG.warning("There is no CRD annotated at the namespace %s",
|
||||
namespace)
|
||||
return
|
||||
net_crd = self._get_net_crd(net_crd_id)
|
||||
|
||||
net_crd_name = 'ns-' + namespace['metadata']['name']
|
||||
|
||||
self._drv_vif_pool.delete_network_pools(net_crd['spec']['netId'])
|
||||
try:
|
||||
|
@ -98,8 +116,7 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
else:
|
||||
LOG.debug("There is no security group associated with the "
|
||||
"namespace to be deleted")
|
||||
|
||||
self._del_kuryrnet_crd(net_crd_id)
|
||||
self._del_kuryrnet_crd(net_crd_name)
|
||||
|
||||
def _get_net_crd_id(self, namespace):
|
||||
try:
|
||||
|
@ -114,6 +131,8 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
try:
|
||||
kuryrnet_crd = k8s.get('%s/kuryrnets/%s' % (constants.K8S_API_CRD,
|
||||
net_crd_id))
|
||||
except exceptions.K8sResourceNotFound:
|
||||
return None
|
||||
except exceptions.K8sClientException:
|
||||
LOG.exception("Kubernetes Client Exception.")
|
||||
raise
|
||||
|
@ -156,6 +175,8 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
try:
|
||||
kubernetes.delete('%s/kuryrnets/%s' % (constants.K8S_API_CRD,
|
||||
net_crd_name))
|
||||
except exceptions.K8sResourceNotFound:
|
||||
LOG.debug("KuryrNetPolicy CRD not found: %s", net_crd_name)
|
||||
except exceptions.K8sClientException:
|
||||
LOG.exception("Kubernetes Client Exception deleting kuryrnet "
|
||||
"CRD.")
|
||||
|
|
|
@ -42,6 +42,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
'selfLink': self._namespace_link},
|
||||
'status': {'phase': 'Active'}
|
||||
}
|
||||
self._crd_id = 'ns-' + self._namespace_name
|
||||
|
||||
self._handler = mock.MagicMock(spec=namespace.NamespaceHandler)
|
||||
|
||||
|
@ -79,6 +80,9 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
def _get_crd(self):
|
||||
crd = {
|
||||
'kind': 'KuryrNet',
|
||||
'metadata': {
|
||||
'selfLink': mock.sentinel.self_link
|
||||
},
|
||||
'spec': {
|
||||
'routerId': mock.sentinel.router_id,
|
||||
'netId': mock.sentinel.net_id,
|
||||
|
@ -114,6 +118,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
def test_on_present(self):
|
||||
net_crd = self._get_crd()
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'test_sg': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'test_sg': 'uuid'}
|
||||
|
@ -122,6 +127,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
@ -142,6 +148,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
|
||||
def test_on_present_create_network_exception(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.side_effect = (
|
||||
n_exc.NeutronClientException)
|
||||
|
||||
|
@ -150,6 +157,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_not_called()
|
||||
|
@ -157,6 +165,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
|
||||
def test_on_present_create_sg_exception(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.side_effect = (
|
||||
n_exc.NeutronClientException)
|
||||
|
@ -166,6 +175,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
@ -174,6 +184,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
|
||||
def test_on_present_add_kuryrnet_crd_exception(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'sgId': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
|
||||
|
@ -182,6 +193,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
@ -195,6 +207,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
net_crd = self._get_crd()
|
||||
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'sgId': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
|
||||
|
@ -204,6 +217,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
@ -217,6 +231,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
net_crd = self._get_crd()
|
||||
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'sgId': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
|
||||
|
@ -230,6 +245,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
@ -249,12 +265,13 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_deleted(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
|
||||
self._get_net_crd.assert_called_once_with(net_crd_id)
|
||||
self._delete_network_pools.assert_called_once_with(
|
||||
net_crd['spec']['netId'])
|
||||
self._delete_namespace_subnet.assert_called_once_with(net_crd)
|
||||
self._delete_sg.assert_called_once_with(net_crd['spec']['sgId'])
|
||||
self._del_kuryrnet_crd.assert_called_once_with(net_crd_id)
|
||||
self._del_kuryrnet_crd.assert_called_once_with(self._crd_id)
|
||||
|
||||
def test_on_deleted_missing_crd_annotation(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
|
|
Loading…
Reference in New Issue