openstack
/
magnum
Code Issues Proposed changes

Allow pod delete to succeed when not found on bay 

If a pod is deleted from kubernetes outside of magnum, and then
a pod-delete is issued through magnum, the delete will succeed
but will leave the record in the database. This change raises
an exception when the pod is not found, but catches it in the
conductor and proceeds to delete the record from the database.

Change-Id: I83a6b3e68eef434fb7d1b4cba92acb17abc6bc4d
Closes-bug: #1412587
This commit is contained in:
Andrew Melton 2015-02-18 09:31:45 -08:00
parent 1197794810
commit a5598a52a9
3 changed files with 148 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
magnum/conductor/handlers/common/kube_utils.py
View File

@ -14,6 +14,7 @@
import tempfile
from magnum.common import exception
from magnum.openstack.common import log as logging
from magnum.openstack.common import utils
@ -191,11 +192,16 @@ class KubeClient(object):
try:
out, err = utils.trycmd('kubectl', 'delete', 'pod', name,
'-s', master_address,)
if err:
return False
except Exception as e:
LOG.error("Couldn't delete pod %s due to error %s" % (name, e))
return False
if err:
if ('pod "%s" not found' % name) in err:
raise exception.PodNotFound(pod=name)
else:
return False
return True
def pod_get(self, master_address, uuid):

67
magnum/tests/conductor/handlers/common/test_kube_utils.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from magnum.common import exception
from magnum.conductor.handlers.common import kube_utils
from magnum.tests import base
@ -142,3 +143,69 @@ class TestKubeUtils(base.BaseTestCase):
mock_file.write.assert_called_once_with(expected_data)
mock_k8s_update.assert_called_once_with(expected_master_address,
expected_filename)
class KubeClientTestCase(base.TestCase):
def setUp(self):
super(KubeClientTestCase, self).setUp()
self.kube_client = kube_utils.KubeClient()
@patch('magnum.openstack.common.utils.trycmd')
def test_pod_delete(self, mock_trycmd):
expected_master_address = 'master-address'
expected_pod_name = 'test-pod'
expected_command = [
'kubectl', 'delete', 'pod', expected_pod_name,
'-s', expected_master_address
]
mock_trycmd.return_value = ("", "")
result = self.kube_client.pod_delete(expected_master_address,
expected_pod_name)
self.assertTrue(result)
mock_trycmd.assert_called_once_with(*expected_command)
@patch('magnum.openstack.common.utils.trycmd')
def test_pod_delete_failure_err_not_empty(self, mock_trycmd):
expected_master_address = 'master-address'
expected_pod_name = 'test-pod'
expected_command = [
'kubectl', 'delete', 'pod', expected_pod_name,
'-s', expected_master_address
]
mock_trycmd.return_value = ("", "error")
result = self.kube_client.pod_delete(expected_master_address,
expected_pod_name)
self.assertFalse(result)
mock_trycmd.assert_called_once_with(*expected_command)
@patch('magnum.openstack.common.utils.trycmd')
def test_pod_delete_failure_exception(self, mock_trycmd):
expected_master_address = 'master-address'
expected_pod_name = 'test-pod'
expected_command = [
'kubectl', 'delete', 'pod', expected_pod_name,
'-s', expected_master_address
]
mock_trycmd.side_effect = Exception()
result = self.kube_client.pod_delete(expected_master_address,
expected_pod_name)
self.assertFalse(result)
mock_trycmd.assert_called_once_with(*expected_command)
@patch('magnum.openstack.common.utils.trycmd')
def test_pod_delete_not_found(self, mock_trycmd):
expected_master_address = 'master-address'
expected_pod_name = 'test-pod'
expected_command = [
'kubectl', 'delete', 'pod', expected_pod_name,
'-s', expected_master_address
]
mock_trycmd.return_value = ("", 'pod "test-pod" not found')
self.assertRaises(exception.PodNotFound, self.kube_client.pod_delete,
expected_master_address, expected_pod_name)
mock_trycmd.assert_called_once_with(*expected_command)

74
magnum/tests/conductor/handlers/test_kube.py
View File

@ -28,7 +28,7 @@ cfg.CONF.import_opt('k8s_port', 'magnum.conductor.handlers.kube',
group='kubernetes')
class TestKube(base.BaseTestCase):
class TestKube(base.TestCase):
def setUp(self):
super(TestKube, self).setUp()
self.kube_handler = kube.Handler()
@ -146,6 +146,78 @@ class TestKube(base.BaseTestCase):
self.kube_handler.pod_create({}, expected_pod)
self.assertEqual('failed', expected_pod.status)
@patch('magnum.conductor.handlers.kube._object_has_stack')
@patch('magnum.conductor.handlers.kube._retrieve_k8s_master_url')
@patch('magnum.objects.Pod.get_by_uuid')
def test_pod_delete_with_success(self,
mock_pod_get_by_uuid,
mock_retrieve_k8s_master_url,
mock_object_has_stack):
expected_master_url = 'master_address'
mock_pod = mock.MagicMock()
mock_pod.name = 'test-pod'
mock_pod.uuid = 'test-uuid'
mock_pod_get_by_uuid.return_value = mock_pod
mock_retrieve_k8s_master_url.return_value = expected_master_url
mock_object_has_stack.return_value = True
with patch.object(self.kube_handler, 'kube_cli') as mock_kube_cli:
mock_kube_cli.pod_delete.return_value = True
self.kube_handler.pod_delete(self.context, mock_pod.uuid)
mock_kube_cli.pod_delete.assert_called_once_with(
expected_master_url, mock_pod.name)
mock_pod.destroy.assert_called_once_with(self.context)
@patch('magnum.conductor.handlers.kube._object_has_stack')
@patch('magnum.conductor.handlers.kube._retrieve_k8s_master_url')
@patch('magnum.objects.Pod.get_by_uuid')
def test_pod_delete_with_failure(self,
mock_pod_get_by_uuid,
mock_retrieve_k8s_master_url,
mock_object_has_stack):
expected_master_url = 'master_address'
mock_pod = mock.MagicMock()
mock_pod.name = 'test-pod'
mock_pod.uuid = 'test-uuid'
mock_pod_get_by_uuid.return_value = mock_pod
mock_retrieve_k8s_master_url.return_value = expected_master_url
mock_object_has_stack.return_value = True
with patch.object(self.kube_handler, 'kube_cli') as mock_kube_cli:
mock_kube_cli.pod_delete.return_value = False
self.kube_handler.pod_delete(self.context, mock_pod.uuid)
mock_kube_cli.pod_delete.assert_called_once_with(
expected_master_url, mock_pod.name)
self.assertFalse(mock_pod.destroy.called)
@patch('magnum.conductor.handlers.kube._object_has_stack')
@patch('magnum.conductor.handlers.kube._retrieve_k8s_master_url')
@patch('magnum.objects.Pod.get_by_uuid')
def test_pod_delete_succeeds_when_not_found(self,
mock_pod_get_by_uuid,
mock_retrieve_k8s_master_url,
mock_object_has_stack):
expected_master_url = 'master_address'
mock_pod = mock.MagicMock()
mock_pod.name = 'test-pod'
mock_pod.uuid = 'test-uuid'
mock_pod_get_by_uuid.return_value = mock_pod
mock_retrieve_k8s_master_url.return_value = expected_master_url
mock_object_has_stack.return_value = True
with patch.object(self.kube_handler, 'kube_cli') as mock_kube_cli:
mock_kube_cli.pod_delete.return_value = True
self.kube_handler.pod_delete(self.context, mock_pod.uuid)
mock_kube_cli.pod_delete.assert_called_once_with(
expected_master_url, mock_pod.name)
mock_pod.destroy.assert_called_once_with(self.context)
@patch('magnum.conductor.handlers.kube._retrieve_k8s_master_url')
def test_service_create_with_success(self,
mock_retrieve_k8s_master_url):