Catch PodNotFound during pod_delete and continue

Previous code wouldn't have completely fixed bug #1412587 as the
pod_delete from the conductor never caught the PodNotFound
exception. This slipped through the tests as the test,
'test_pod_delete_succeeds_when_not_found', was not set up properly.

This code fixes the test and implements a full fix.

Change-Id: I54cfc9f657179a63adb28a6986e528a74a6eeff6
Closes-bug: #1424473

magnum/conductor/handlers/kube.py

@@ -15,6 +15,7 @@
 from oslo_config import cfg
 
 from magnum.common import clients
+from magnum.common import exception
 from magnum.conductor.handlers.common import kube_utils
 from magnum import objects
 from magnum.openstack.common._i18n import _
@@ -174,9 +175,15 @@ class Handler(object):
         pod = objects.Pod.get_by_uuid(context, uuid)
         k8s_master_url = _retrieve_k8s_master_url(context, pod)
         if _object_has_stack(context, pod):
-            status = self.kube_cli.pod_delete(k8s_master_url, pod.name)
-            if not status:
-                return None
+            try:
+                status = self.kube_cli.pod_delete(k8s_master_url, pod.name)
+
+                if not status:
+                    return None
+            except exception.PodNotFound:
+                msg = ("Pod '%s' not found on bay, "
+                       "continuing to delete from database.")
+                LOG.warn(msg, uuid)
         # call the pod object to persist in db
         pod.destroy(context)
 

magnum/tests/conductor/handlers/test_kube.py

@@ -14,6 +14,7 @@
 
 from oslo_config import cfg
 
+from magnum.common import exception
 from magnum.conductor.handlers import kube
 from magnum import objects
 from magnum.tests import base
@@ -210,7 +211,7 @@ class TestKube(base.TestCase):
         mock_retrieve_k8s_master_url.return_value = expected_master_url
         mock_object_has_stack.return_value = True
         with patch.object(self.kube_handler, 'kube_cli') as mock_kube_cli:
-            mock_kube_cli.pod_delete.return_value = True
+            mock_kube_cli.pod_delete.side_effect = exception.PodNotFound()
 
             self.kube_handler.pod_delete(self.context, mock_pod.uuid)