Browse Source

NP: Protect from disappearing resources

It may happen that between we list some resources and start iterating
through them, some get deleted. This commit makes sure we're ignoring
errors coming from such situations in NP code.

Closes-Bug: 1894194

Change-Id: I082ab9d5881eab5a4686f4f3ec43b1cd0d8e8ad8
changes/06/749906/1
Michał Dulko 11 months ago
parent
commit
561f384789
  1. 2
      kuryr_kubernetes/controller/drivers/network_policy.py
  2. 25
      kuryr_kubernetes/controller/drivers/network_policy_security_groups.py
  3. 11
      kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py

2
kuryr_kubernetes/controller/drivers/network_policy.py

@ -609,7 +609,7 @@ class NetworkPolicyDriver(base.NetworkPolicyDriver):
try:
self.os_net.delete_security_group(sg_id)
except os_exc.ConflictException:
LOG.debug("Security Group already in use: %s", sg_id)
LOG.debug("Security Group %s still in use!", sg_id)
# raising ResourceNotReady to retry this action in case ports
# associated to affected pods are not updated on time, i.e.,
# they are still using the security group to be removed

25
kuryr_kubernetes/controller/drivers/network_policy_security_groups.py

@ -52,7 +52,6 @@ def _bump_networkpolicy(knp):
knp['metadata']['annotations']['networkPolicyLink'],
{constants.K8S_ANNOTATION_POLICY: str(uuid.uuid4())})
except exceptions.K8sResourceNotFound:
LOG.exception("NetworkPolicy not found")
raise
except exceptions.K8sClientException:
LOG.exception("Kubernetes Client Exception")
@ -321,7 +320,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
e_matched = _parse_rules('egress', crd, spec, pod=pod)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_pod_selectors.append(crd_selector)
return crd_pod_selectors
@ -346,7 +349,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
egress_rule_list, "egress", pod_ip)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_pod_selectors.append(crd_selector)
return crd_pod_selectors
@ -376,7 +383,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
egress_rule_list, "egress", ns_name)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_selectors.append(crd_selector)
return crd_selectors
@ -395,7 +406,11 @@ class NetworkPolicySecurityGroupsDriver(base.PodSecurityGroupsDriver):
e_matched = _parse_rules('egress', crd, spec, namespace=namespace)
if i_matched or e_matched:
_bump_networkpolicy(crd)
try:
_bump_networkpolicy(crd)
except exceptions.K8sResourceNotFound:
# The NP got deleted, ignore it.
continue
if i_matched:
crd_selectors.append(crd_selector)
return crd_selectors

11
kuryr_kubernetes/controller/handlers/kuryrnetworkpolicy.py

@ -196,7 +196,11 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
if driver_utils.is_host_network(pod):
continue
pod_sgs = self._drv_pod_sg.get_security_groups(pod, project_id)
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
try:
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
except os_exc.NotFoundException:
# Pod got deleted in the meanwhile, should be safe to ignore.
pass
# FIXME(dulek): We should not need this one day.
policy = self._get_networkpolicy(knp['metadata']['annotations']
@ -279,9 +283,8 @@ class KuryrNetworkPolicyHandler(k8s_base.ResourceEventHandler):
try:
self._drv_vif_pool.update_vif_sgs(pod, pod_sgs)
except os_exc.NotFoundException:
LOG.debug("Fail to update pod sgs."
" Retrying policy deletion.")
raise exceptions.ResourceNotReady(knp)
# Pod got deleted in the meanwhile, safe to ignore.
pass
# ensure ports at the pool don't have the NP sg associated
try:

Loading…
Cancel
Save