Services: Rollback openstack resources in case of annotation failure
Upon K8S service creation the LBaaS handler creates all LB resources at neutron (LB,Listener,Pool,etc) and store them at K8S resource using annotation. When K8S service is deleted, the LBaaS handler retrieves LB resources details from annotation and release them at neutron. This patch handles the case in which K8S service resource was deleted before LBaaS handler stored openstack resource details. Closes-Bug: 1748890 Change-Id: Iea806d32c99cd3cf51a832b576ff4054fc522bd3
This commit is contained in:
parent
48baed57d9
commit
2e6c7eaae7
@ -246,9 +246,18 @@ class LoadBalancerHandler(k8s_base.ResourceEventHandler):
|
|||||||
# required to deal with such situations (e.g. cleanup, or skip
|
# required to deal with such situations (e.g. cleanup, or skip
|
||||||
# failing items, or validate configuration) to prevent annotation
|
# failing items, or validate configuration) to prevent annotation
|
||||||
# being out of sync with the actual Neutron state.
|
# being out of sync with the actual Neutron state.
|
||||||
|
try:
|
||||||
self._set_lbaas_state(endpoints, lbaas_state)
|
self._set_lbaas_state(endpoints, lbaas_state)
|
||||||
|
except k_exc.K8sResourceNotFound:
|
||||||
|
# Note(yboaron) It's impossible to store neutron resources
|
||||||
|
# in K8S object since object was deleted. In that case
|
||||||
|
# we should rollback all neutron resources.
|
||||||
|
LOG.debug("LoadBalancerHandler failed to store Openstack "
|
||||||
|
"resources in K8S object (not found)")
|
||||||
|
self.on_deleted(endpoints, lbaas_state)
|
||||||
|
|
||||||
def on_deleted(self, endpoints):
|
def on_deleted(self, endpoints, lbaas_state=None):
|
||||||
|
if lbaas_state is None:
|
||||||
lbaas_state = self._get_lbaas_state(endpoints)
|
lbaas_state = self._get_lbaas_state(endpoints)
|
||||||
if not lbaas_state:
|
if not lbaas_state:
|
||||||
return
|
return
|
||||||
|
@ -28,6 +28,12 @@ class ResourceNotReady(Exception):
|
|||||||
% resource)
|
% resource)
|
||||||
|
|
||||||
|
|
||||||
|
class K8sResourceNotFound(K8sClientException):
|
||||||
|
def __init__(self, resource):
|
||||||
|
super(K8sResourceNotFound, self).__init__("Resource not "
|
||||||
|
"found: %r" % resource)
|
||||||
|
|
||||||
|
|
||||||
class CNIError(Exception):
|
class CNIError(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -145,6 +145,10 @@ class K8sClient(object):
|
|||||||
"content: %(content)s, text: %(text)s"
|
"content: %(content)s, text: %(text)s"
|
||||||
% {'headers': response.headers,
|
% {'headers': response.headers,
|
||||||
'content': response.content, 'text': response.text})
|
'content': response.content, 'text': response.text})
|
||||||
|
|
||||||
|
if response.status_code == requests.codes.not_found:
|
||||||
|
raise exc.K8sResourceNotFound(response.text)
|
||||||
|
else:
|
||||||
raise exc.K8sClientException(response.text)
|
raise exc.K8sClientException(response.text)
|
||||||
|
|
||||||
def watch(self, path):
|
def watch(self, path):
|
||||||
|
@ -423,6 +423,31 @@ class TestLoadBalancerHandler(test_base.TestCase):
|
|||||||
m_handler._set_lbaas_state.assert_called_once_with(
|
m_handler._set_lbaas_state.assert_called_once_with(
|
||||||
endpoints, lbaas_state)
|
endpoints, lbaas_state)
|
||||||
|
|
||||||
|
def test_on_present_rollback(self):
|
||||||
|
lbaas_spec = mock.sentinel.lbaas_spec
|
||||||
|
lbaas_state = mock.sentinel.lbaas_state
|
||||||
|
endpoints = mock.sentinel.endpoints
|
||||||
|
|
||||||
|
m_handler = mock.Mock(spec=h_lbaas.LoadBalancerHandler)
|
||||||
|
m_handler._get_lbaas_spec.return_value = lbaas_spec
|
||||||
|
m_handler._should_ignore.return_value = False
|
||||||
|
m_handler._get_lbaas_state.return_value = lbaas_state
|
||||||
|
m_handler._sync_lbaas_members.return_value = True
|
||||||
|
m_handler._set_lbaas_state.side_effect = (
|
||||||
|
k_exc.K8sResourceNotFound('ep'))
|
||||||
|
|
||||||
|
h_lbaas.LoadBalancerHandler.on_present(m_handler, endpoints)
|
||||||
|
|
||||||
|
m_handler._get_lbaas_spec.assert_called_once_with(endpoints)
|
||||||
|
m_handler._should_ignore.assert_called_once_with(endpoints, lbaas_spec)
|
||||||
|
m_handler._get_lbaas_state.assert_called_once_with(endpoints)
|
||||||
|
m_handler._sync_lbaas_members.assert_called_once_with(
|
||||||
|
endpoints, lbaas_state, lbaas_spec)
|
||||||
|
m_handler._set_lbaas_state.assert_called_once_with(
|
||||||
|
endpoints, lbaas_state)
|
||||||
|
m_handler.on_deleted.assert_called_once_with(
|
||||||
|
endpoints, lbaas_state)
|
||||||
|
|
||||||
@mock.patch('kuryr_kubernetes.objects.lbaas'
|
@mock.patch('kuryr_kubernetes.objects.lbaas'
|
||||||
'.LBaaSServiceSpec')
|
'.LBaaSServiceSpec')
|
||||||
def test_on_deleted(self, m_svc_spec_ctor):
|
def test_on_deleted(self, m_svc_spec_ctor):
|
||||||
|
@ -280,6 +280,33 @@ class TestK8sClient(test_base.TestCase):
|
|||||||
headers=mock.ANY,
|
headers=mock.ANY,
|
||||||
cert=(None, None), verify=False)
|
cert=(None, None), verify=False)
|
||||||
|
|
||||||
|
@mock.patch('itertools.count')
|
||||||
|
@mock.patch('requests.patch')
|
||||||
|
def test_annotate_resource_not_found(self, m_patch, m_count):
|
||||||
|
m_count.return_value = list(range(1, 5))
|
||||||
|
path = '/test'
|
||||||
|
annotations = {'a1': 'v1', 'a2': 'v2'}
|
||||||
|
resource_version = "123"
|
||||||
|
annotate_obj = {'metadata': {
|
||||||
|
'annotations': annotations,
|
||||||
|
'resourceVersion': resource_version}}
|
||||||
|
annotate_data = jsonutils.dumps(annotate_obj, sort_keys=True)
|
||||||
|
|
||||||
|
m_resp_not_found = mock.MagicMock()
|
||||||
|
m_resp_not_found.ok = False
|
||||||
|
m_resp_not_found.status_code = requests.codes.not_found
|
||||||
|
m_patch.return_value = m_resp_not_found
|
||||||
|
|
||||||
|
self.assertRaises(exc.K8sResourceNotFound,
|
||||||
|
self.client.annotate,
|
||||||
|
path,
|
||||||
|
annotations,
|
||||||
|
resource_version=resource_version)
|
||||||
|
m_patch.assert_called_once_with(self.base_url + path,
|
||||||
|
data=annotate_data,
|
||||||
|
headers=mock.ANY,
|
||||||
|
cert=(None, None), verify=False)
|
||||||
|
|
||||||
@mock.patch('requests.get')
|
@mock.patch('requests.get')
|
||||||
def test_watch(self, m_get):
|
def test_watch(self, m_get):
|
||||||
path = '/test'
|
path = '/test'
|
||||||
|
Loading…
Reference in New Issue
Block a user