openstack
/
kuryr-kubernetes
Code Issues Proposed changes

NP: Protect from disappearing resources 

It may happen that between we list some resources and start iterating
through them, some get deleted. This commit makes sure we're ignoring
errors coming from such situations in NP code.

Closes-Bug: 1894194

Change-Id: I082ab9d5881eab5a4686f4f3ec43b1cd0d8e8ad8
This commit is contained in:
Michał Dulko 2020-08-27 13:14:21 +02:00
parent a8e97569a8
commit 561f384789
3 changed files with 28 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
kuryr_kubernetes/controller/drivers/network_policy.py
View File

@ -609,7 +609,7 @@ class NetworkPolicyDriver(base.NetworkPolicyDriver):
try: try:
self.os_net.delete_security_group(sg_id) self.os_net.delete_security_group(sg_id)
except os_exc.ConflictException: except os_exc.ConflictException:
LOG.debug("Security Group already in use: %s", sg_id) LOG.debug("Security Group %s still in use!", sg_id)
# raising ResourceNotReady to retry this action in case ports # raising ResourceNotReady to retry this action in case ports
# associated to affected pods are not updated on time, i.e., # associated to affected pods are not updated on time, i.e.,
# they are still using the security group to be removed # they are still using the security group to be removed

25
kuryr_kubernetes/controller/drivers/network_policy_security_groups.py
View File

@ -52,7 +52,6 @@ def _bump_networkpolicy(knp):
knp['metadata']['annotations']['networkPolicyLink'], knp['metadata']['annotations']['networkPolicyLink'],
{constants.K8S_ANNOTATION_POLICY: str(uuid.uuid4())}) {constants.K8S_ANNOTATION_POLICY: str(uuid.uuid4())})
except exceptions.K8sResourceNotFound: except exceptions.K8sResourceNotFound:
LOG.exception("NetworkPolicy not found")
raise raise
except exceptions.K8sClientException: except exceptions.K8sClientException:
LOG.exception("Kubernetes Client Exception") LOG.exception("Kubernetes Client Exception")
@ -321,7 +320,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
e_matched = _parse_rules('egress', crd, spec, pod=pod) e_matched = _parse_rules('egress', crd, spec, pod=pod)
if i_matched or e_matched: if i_matched or e_matched:
_bump_networkpolicy(crd) try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched: if i_matched:
crd_pod_selectors.append(crd_selector) crd_pod_selectors.append(crd_selector)
return crd_pod_selectors return crd_pod_selectors
@ -346,7 +349,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
egress_rule_list, "egress", pod_ip) egress_rule_list, "egress", pod_ip)
if i_matched or e_matched: if i_matched or e_matched:
_bump_networkpolicy(crd) try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched: if i_matched:
crd_pod_selectors.append(crd_selector) crd_pod_selectors.append(crd_selector)
return crd_pod_selectors return crd_pod_selectors
@ -376,7 +383,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
egress_rule_list, "egress", ns_name) egress_rule_list, "egress", ns_name)
if i_matched or e_matched: if i_matched or e_matched:
_bump_networkpolicy(crd) try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched: if i_matched:
crd_selectors.append(crd_selector) crd_selectors.append(crd_selector)
return crd_selectors return crd_selectors
@ -395,7 +406,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
e_matched = _parse_rules('egress', crd, spec, namespace=namespace) e_matched = _parse_rules('egress', crd, spec, namespace=namespace)
if i_matched or e_matched: if i_matched or e_matched:
_bump_networkpolicy(crd) try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched: if i_matched:
crd_selectors.append(crd_selector) crd_selectors.append(crd_selector)
return crd_selectors return crd_selectors

11
kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py
View File

@ -196,7 +196,11 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
if driver_utils.is_host_network(pod): if driver_utils.is_host_network(pod):
continue continue
pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id) pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id)
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) try:
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
except os_exc.NotFoundException:
# Pod got deleted in the meanwhile, should be safe to ignore.
pass
# FIXME(dulek): We should not need this one day. # FIXME(dulek): We should not need this one day.
policy = self._get_networkpolicy(knp['metadata']['annotations'] policy = self._get_networkpolicy(knp['metadata']['annotations']
@ -279,9 +283,8 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
try: try:
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs) self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
except os_exc.NotFoundException: except os_exc.NotFoundException:
LOG.debug("Fail to update pod sgs." # Pod got deleted in the meanwhile, safe to ignore.
" Retrying policy deletion.") pass
raise exceptions.ResourceNotReady(knp)
# ensure ports at the pool don't have the NP sg associated # ensure ports at the pool don't have the NP sg associated
try: try: