NP: Protect from disappearing resources

It may happen that between we list some resources and start iterating
through them, some get deleted. This commit makes sure we're ignoring
errors coming from such situations in NP code.

Closes-Bug: 1894194

Change-Id: I082ab9d5881eab5a4686f4f3ec43b1cd0d8e8ad8
This commit is contained in:
Michał Dulko 2020-08-27 13:14:21 +02:00
parent a8e97569a8
commit 561f384789
3 changed files with 28 additions and 10 deletions

View File

@ -609,7 +609,7 @@ class NetworkPolicyDriver(base.NetworkPolicyDriver):
try:
self.os_net.delete_security_group(sg_id)
except os_exc.ConflictException:
LOG.debug("Security Group already in use: %s", sg_id)
LOG.debug("Security Group %s still in use!", sg_id)
# raising ResourceNotReady to retry this action in case ports
# associated to affected pods are not updated on time, i.e.,
# they are still using the security group to be removed

View File

@ -52,7 +52,6 @@ def _bump_networkpolicy(knp):
knp['metadata']['annotations']['networkPolicyLink'],
{constants.K8S_ANNOTATION_POLICY: str(uuid.uuid4())})
except exceptions.K8sResourceNotFound:
LOG.exception("NetworkPolicy not found")
raise
except exceptions.K8sClientException:
LOG.exception("Kubernetes Client Exception")
@ -321,7 +320,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
e_matched = _parse_rules('egress', crd, spec, pod=pod)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_pod_selectors.append(crd_selector)
return crd_pod_selectors
@ -346,7 +349,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
egress_rule_list, "egress", pod_ip)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_pod_selectors.append(crd_selector)
return crd_pod_selectors
@ -376,7 +383,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
egress_rule_list, "egress", ns_name)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_selectors.append(crd_selector)
return crd_selectors
@ -395,7 +406,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
e_matched = _parse_rules('egress', crd, spec, namespace=namespace)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_selectors.append(crd_selector)
return crd_selectors

View File

@ -196,7 +196,11 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
if driver_utils.is_host_network(pod):
continue
pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id)
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
try:
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
except os_exc.NotFoundException:
# Pod got deleted in the meanwhile, should be safe to ignore.
pass
# FIXME(dulek): We should not need this one day.
policy = self._get_networkpolicy(knp['metadata']['annotations']
@ -279,9 +283,8 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
try:
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
except os_exc.NotFoundException:
LOG.debug("Fail to update pod sgs."
" Retrying policy deletion.")
raise exceptions.ResourceNotReady(knp)
# Pod got deleted in the meanwhile, safe to ignore.
pass
# ensure ports at the pool don't have the NP sg associated
try: