Guard against manually removing of KuryrPort CRD.

It might happen, that operator would manually removing KuryrPort CRD
corresponding to the pod, and than pod would become broken  - there
would be no connectivity to it.

In this patch we prevent from removing KP, in case Pod is not in
deleting state. Now, it will defer the deletion till the Pod deletion.

Change-Id: Idd6383296723c41ed6f29715969c452ef3fcdb1f

kuryr_kubernetes/controller/handlers/kuryrport.py

@@ -137,6 +137,15 @@ class KuryrPortHandler(k8s_base.ResourceEventHandler):
                                       constants.KURYRPORT_FINALIZER)
             return
 
+        if 'deletionTimestamp' not in pod['metadata']:
+            # NOTE(gryf): Ignore deleting KuryrPort, since most likely it was
+            # removed manually, while we need vifs for corresponding pod
+            # object which apperantely is still running.
+            LOG.warning('Manually triggered KuryrPort %s removal. This '
+                        'action should be avoided, since KuryrPort CRDs are '
+                        'internal to Kuryr.', name)
+            return
+
         project_id = self._drv_project.get_project(pod)
         try:
             crd_pod_selectors = self._drv_sg.delete_sg_rules(pod)

kuryr_kubernetes/controller/handlers/vif.py

@@ -105,16 +105,38 @@ class VIFHandler(k8s_base.ResourceEventHandler):
 
     def on_finalize(self, pod):
         k8s = clients.get_kubernetes_client()
+
         try:
-            k8s.delete(KURYRPORT_URI.format(ns=pod["metadata"]["namespace"],
-                                            crd=pod["metadata"]["name"]))
+            kp = k8s.get(KURYRPORT_URI.format(ns=pod["metadata"]["namespace"],
+                                              crd=pod["metadata"]["name"]))
         except k_exc.K8sResourceNotFound:
             k8s.remove_finalizer(pod, constants.POD_FINALIZER)
+            return
 
-        except k_exc.K8sClientException:
-            LOG.exception("Could not remove KuryrPort CRD for pod %s.",
-                          pod['metadata']['name'])
-            raise k_exc.ResourceNotReady(pod['metadata']['name'])
+        if 'deletionTimestamp' in kp['metadata']:
+            # NOTE(gryf): Seems like KP was manually removed. By using
+            # annotations, force an emition of event to trigger on_finalize
+            # method on the KuryrPort.
+            try:
+                k8s.annotate(kp['metadata']['selfLink'], {'KuryrTrigger': '1'})
+            except k_exc.K8sResourceNotFound:
+                LOG.error('Cannot annotate existing KuryrPort %s.',
+                          kp['metadata']['name'])
+                k8s.remove_finalizer(pod, constants.POD_FINALIZER)
+            except k_exc.K8sClientException:
+                raise k_exc.ResourceNotReady(pod['metadata']['name'])
+        else:
+            try:
+                k8s.delete(KURYRPORT_URI
+                           .format(ns=pod["metadata"]["namespace"],
+                                   crd=pod["metadata"]["name"]))
+            except k_exc.K8sResourceNotFound:
+                k8s.remove_finalizer(pod, constants.POD_FINALIZER)
+
+            except k_exc.K8sClientException:
+                LOG.exception("Could not remove KuryrPort CRD for pod %s.",
+                              pod['metadata']['name'])
+                raise k_exc.ResourceNotReady(pod['metadata']['name'])
 
     def is_ready(self, quota):
         if (utils.has_limit(quota.ports) and

kuryr_kubernetes/tests/unit/controller/handlers/test_kuryrport.py

@@ -48,6 +48,7 @@ class TestKuryrPortHandler(test_base.TestCase):
         self._pod = {'metadata': {'resourceVersion': self._pod_version,
                                   'selfLink': self._pod_link,
                                   'name': self._kp_name,
+                                  'deletionTimestamp': mock.sentinel.date,
                                   'namespace': self._kp_namespace},
                      'spec': {'nodeName': self._host}}
 
@@ -440,6 +441,22 @@ class TestKuryrPortHandler(test_base.TestCase):
         update_services.assert_called_once_with(mock.sentinel.services,
                                                 selector, self._project_id)
 
+    @mock.patch('kuryr_kubernetes.clients.get_kubernetes_client')
+    @mock.patch('kuryr_kubernetes.controller.drivers.base.MultiVIFDriver.'
+                'get_enabled_drivers')
+    def test_on_finalize_pod_running(self, ged, k8s):
+        ged.return_value = [self._driver]
+        # copy, so it will not be affected by other tests run in parallel.
+        pod = dict(self._pod)
+        del(pod['metadata']['deletionTimestamp'])
+
+        kp = kuryrport.KuryrPortHandler()
+
+        with mock.patch.object(kp, 'k8s') as k8s:
+            k8s.get.return_value = pod
+            self.assertIsNone(kp.on_finalize(self._kp))
+            k8s.get.assert_called_once_with(self._pod_uri)
+
     @mock.patch('kuryr_kubernetes.controller.handlers.kuryrport.'
                 'KuryrPortHandler._update_kuryrport_crd')
     @mock.patch('kuryr_kubernetes.controller.drivers.vif_pool.MultiVIFPool.'