Fix fail to recreate namespace when previous KuryrNet CRD is not deleted
When a namespace/project is recreated right after its deletion, it's possible that the previous kuryrNet CRD is not yet removed, and Kuryr will try to create another CRD, resulting in k8s conflict exception. In the meantime, other events handling resources under the same namespace in question could be inquiring for an annotation on the namespace, that was not annotated due to the conflict error, causing the controller to restart. This commit fixes the issue by adding an annotation to the KuryrNet CRD with the respective namespace uid, which allows to check if the existing CRD corresponds to the namespace. Change-Id: I85d903902b516bfd109078705860540d731886bc Closes-Bug: 1833712
This commit is contained in:
Maysa Macedo
parent
e24126be08
commit
37b8e3e6b3
|
|
@ -59,6 +59,7 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
self._drv_vif_pool.set_vif_driver()
|
||||
|
||||
def on_present(self, namespace):
|
||||
ns_name = namespace['metadata']['name']
|
||||
current_namespace_labels = namespace['metadata'].get('labels')
|
||||
previous_namespace_labels = drivers_utils.get_annotated_labels(
|
||||
namespace, constants.K8S_ANNOTATION_NAMESPACE_LABEL)
|
||||
|
|
@ -69,13 +70,20 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
self._drv_sg.update_namespace_sg_rules(namespace)
|
||||
self._set_namespace_labels(namespace, current_namespace_labels)
|
||||
|
||||
ns_name = namespace['metadata']['name']
|
||||
project_id = self._drv_project.get_project(namespace)
|
||||
net_crd_id = self._get_net_crd_id(namespace)
|
||||
if net_crd_id:
|
||||
LOG.debug("CRD existing at the new namespace")
|
||||
return
|
||||
|
||||
net_crd_name = 'ns-' + ns_name
|
||||
net_crd = self._get_net_crd(net_crd_name)
|
||||
if net_crd:
|
||||
LOG.debug("Previous CRD existing at the new namespace. "
|
||||
"Deleting namespace resources and retying its creation.")
|
||||
self.on_deleted(namespace, net_crd)
|
||||
raise exceptions.ResourceNotReady(namespace)
|
||||
|
||||
LOG.debug("Creating network resources for namespace: %s", ns_name)
|
||||
net_crd_spec = self._drv_subnets.create_namespace_network(ns_name,
|
||||
project_id)
|
||||
|
|
@ -99,20 +107,29 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
self._set_net_crd(namespace, net_crd)
|
||||
self._drv_sg.create_namespace_sg_rules(namespace)
|
||||
self._set_namespace_labels(namespace, current_namespace_labels)
|
||||
except exceptions.K8sResourceNotFound:
|
||||
LOG.debug("Namespace could not be annotated. Rolling back "
|
||||
"resources created for it.")
|
||||
self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name)
|
||||
self._drv_sg.delete_sg(net_crd_sg['sgId'])
|
||||
self._del_kuryrnet_crd(net_crd_name)
|
||||
except exceptions.K8sClientException:
|
||||
LOG.exception("Kuryrnet CRD could not be added. Rolling back "
|
||||
"network resources created for the namespace.")
|
||||
self._drv_subnets.rollback_network_resources(net_crd_spec, ns_name)
|
||||
self._drv_sg.delete_sg(net_crd_sg['sgId'])
|
||||
|
||||
def on_deleted(self, namespace):
|
||||
def on_deleted(self, namespace, net_crd=None):
|
||||
LOG.debug("Deleting namespace: %s", namespace)
|
||||
net_crd_id = self._get_net_crd_id(namespace)
|
||||
if not net_crd_id:
|
||||
LOG.warning("There is no CRD annotated at the namespace %s",
|
||||
namespace)
|
||||
return
|
||||
net_crd = self._get_net_crd(net_crd_id)
|
||||
if not net_crd:
|
||||
net_crd_id = self._get_net_crd_id(namespace)
|
||||
if not net_crd_id:
|
||||
LOG.warning("There is no CRD annotated at the namespace %s",
|
||||
namespace)
|
||||
return
|
||||
net_crd = self._get_net_crd(net_crd_id)
|
||||
|
||||
net_crd_name = 'ns-' + namespace['metadata']['name']
|
||||
|
||||
self._drv_vif_pool.delete_network_pools(net_crd['spec']['netId'])
|
||||
try:
|
||||
|
|
@ -130,7 +147,7 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
else:
|
||||
LOG.debug("There is no security group associated with the "
|
||||
"namespace to be deleted")
|
||||
self._del_kuryrnet_crd(net_crd_id)
|
||||
self._del_kuryrnet_crd(net_crd_name)
|
||||
self._drv_sg.delete_namespace_sg_rules(namespace)
|
||||
|
||||
def is_ready(self, quota):
|
||||
|
|
@ -167,6 +184,8 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
try:
|
||||
kuryrnet_crd = k8s.get('%s/kuryrnets/%s' % (constants.K8S_API_CRD,
|
||||
net_crd_id))
|
||||
except exceptions.K8sResourceNotFound:
|
||||
return None
|
||||
except exceptions.K8sClientException:
|
||||
LOG.exception("Kubernetes Client Exception.")
|
||||
raise
|
||||
|
|
@ -214,6 +233,8 @@ class NamespaceHandler(k8s_base.ResourceEventHandler):
|
|||
try:
|
||||
kubernetes.delete('%s/kuryrnets/%s' % (constants.K8S_API_CRD,
|
||||
net_crd_name))
|
||||
except exceptions.K8sResourceNotFound:
|
||||
LOG.debug("KuryrNetPolicy CRD not found: %s", net_crd_name)
|
||||
except exceptions.K8sClientException:
|
||||
LOG.exception("Kubernetes Client Exception deleting kuryrnet "
|
||||
"CRD.")
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
'selfLink': self._namespace_link},
|
||||
'status': {'phase': 'Active'}
|
||||
}
|
||||
self._crd_id = 'ns-' + self._namespace_name
|
||||
|
||||
self._handler = mock.MagicMock(spec=namespace.NamespaceHandler)
|
||||
|
||||
|
|
@ -79,6 +80,9 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
def _get_crd(self):
|
||||
crd = {
|
||||
'kind': 'KuryrNet',
|
||||
'metadata': {
|
||||
'selfLink': mock.sentinel.self_link
|
||||
},
|
||||
'spec': {
|
||||
'routerId': mock.sentinel.router_id,
|
||||
'netId': mock.sentinel.net_id,
|
||||
|
|
@ -114,6 +118,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
def test_on_present(self):
|
||||
net_crd = self._get_crd()
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'test_sg': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'test_sg': 'uuid'}
|
||||
|
|
@ -122,6 +127,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
|
@ -142,6 +148,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
|
||||
def test_on_present_create_network_exception(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.side_effect = (
|
||||
n_exc.NeutronClientException)
|
||||
|
||||
|
|
@ -150,6 +157,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_not_called()
|
||||
|
|
@ -157,6 +165,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
|
||||
def test_on_present_create_sg_exception(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.side_effect = (
|
||||
n_exc.NeutronClientException)
|
||||
|
|
@ -166,6 +175,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
|
@ -174,6 +184,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
|
||||
def test_on_present_add_kuryrnet_crd_exception(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'sgId': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
|
||||
|
|
@ -182,6 +193,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
|
@ -195,6 +207,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
net_crd = self._get_crd()
|
||||
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'sgId': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
|
||||
|
|
@ -204,6 +217,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_present(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
|
@ -217,6 +231,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
net_crd = self._get_crd()
|
||||
|
||||
self._get_net_crd_id.return_value = None
|
||||
self._get_net_crd.return_value = None
|
||||
self._create_namespace_network.return_value = {'test_net': 'uuid'}
|
||||
self._create_namespace_sg.return_value = {'sgId': 'uuid'}
|
||||
net_crd_spec = {'test_net': 'uuid', 'sgId': 'uuid'}
|
||||
|
|
@ -230,6 +245,7 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
self._get_net_crd.assert_called_once_with(self._crd_id)
|
||||
self._create_namespace_network.assert_called_once_with(
|
||||
self._namespace_name, self._project_id)
|
||||
self._create_namespace_sg.assert_called_once_with(
|
||||
|
|
@ -249,12 +265,13 @@ class TestNamespaceHandler(test_base.TestCase):
|
|||
namespace.NamespaceHandler.on_deleted(self._handler, self._namespace)
|
||||
|
||||
self._get_net_crd_id.assert_called_once_with(self._namespace)
|
||||
|
||||
self._get_net_crd.assert_called_once_with(net_crd_id)
|
||||
self._delete_network_pools.assert_called_once_with(
|
||||
net_crd['spec']['netId'])
|
||||
self._delete_namespace_subnet.assert_called_once_with(net_crd)
|
||||
self._delete_sg.assert_called_once_with(net_crd['spec']['sgId'])
|
||||
self._del_kuryrnet_crd.assert_called_once_with(net_crd_id)
|
||||
self._del_kuryrnet_crd.assert_called_once_with(self._crd_id)
|
||||
|
||||
def test_on_deleted_missing_crd_annotation(self):
|
||||
self._get_net_crd_id.return_value = None
|
||||
|
|
|
|||
Loading…
Reference in New Issue