Merge "Ensure controller is only restarted after the event timesout"

kuryr_kubernetes/controller/drivers/vif_pool.py

@@ -192,8 +192,8 @@ class BaseVIFPool(base.VIFPoolDriver):
         try:
             host_addr = self._get_host_addr(pod)
         except KeyError:
-            LOG.warning("Pod has not been scheduled yet.")
-            raise
+            return None
+
         pool_key = self._get_pool_key(host_addr, project_id, None, subnets)
 
         try:

kuryr_kubernetes/controller/handlers/lbaas.py

@@ -288,7 +288,15 @@ class LoadBalancerHandler(k8s_base.ResourceEventHandler):
 
         lb = lbaas_state.loadbalancer
         default_sgs = config.CONF.neutron_defaults.pod_security_groups
-        lbaas_spec_sgs = lbaas_spec.security_groups_ids
+        # NOTE(maysams) As the endpoint and svc are annotated with the
+        # 'lbaas_spec' in two separate k8s calls, it's possible that
+        # the endpoint got annotated and the svc haven't due to controller
+        # restarts. For this case, a resourceNotReady exception is raised
+        # till the svc gets annotated with a 'lbaas_spec'.
+        if lbaas_spec:
+            lbaas_spec_sgs = lbaas_spec.security_groups_ids
+        else:
+            raise k_exc.ResourceNotReady(svc_link)
         if lb.security_groups and lb.security_groups != lbaas_spec_sgs:
             sgs = [lb_sg for lb_sg in lb.security_groups
                    if lb_sg not in default_sgs]

kuryr_kubernetes/controller/handlers/vif.py

@@ -88,14 +88,20 @@ class VIFHandler(k8s_base.ResourceEventHandler):
         state = driver_utils.get_pod_state(pod)
         LOG.debug("Got VIFs from annotation: %r", state)
         project_id = self._drv_project.get_project(pod)
+        security_groups = self._drv_sg.get_security_groups(pod, project_id)
         if not state:
-            security_groups = self._drv_sg.get_security_groups(pod, project_id)
             subnets = self._drv_subnets.get_subnets(pod, project_id)
 
             # Request the default interface of pod
             main_vif = self._drv_vif_pool.request_vif(
                 pod, project_id, subnets, security_groups)
 
+            if not main_vif:
+                pod_name = pod['metadata']['name']
+                LOG.warning("Ignoring event due to pod %s not being "
+                            "scheduled yet.", pod_name)
+                return
+
             state = objects.vif.PodState(default_vif=main_vif)
 
             # Request the additional interfaces from multiple dirvers
@@ -131,7 +137,17 @@ class VIFHandler(k8s_base.ResourceEventHandler):
                         changed = True
             finally:
                 if changed:
-                    self._set_pod_state(pod, state)
+                    try:
+                        self._set_pod_state(pod, state)
+                    except k_exc.K8sResourceNotFound as ex:
+                        LOG.exception("Failed to set annotation: %s", ex)
+                        for ifname, vif in state.vifs.items():
+                            self._drv_vif_pool.release_vif(
+                                pod, vif, project_id,
+                                security_groups)
+                    except k_exc.K8sClientException:
+                        pod_name = pod['metadata']['name']
+                        raise k_exc.ResourceNotReady(pod_name)
                     if self._is_network_policy_enabled():
                         crd_pod_selectors = self._drv_sg.create_sg_rules(pod)
                         if oslo_cfg.CONF.octavia_defaults.enforce_sg_rules:

kuryr_kubernetes/tests/unit/controller/drivers/test_vif_pool.py

@@ -170,8 +170,9 @@ class BaseVIFPool(test_base.TestCase):
         security_groups = [mock.sentinel.security_groups]
         m_driver._get_host_addr.side_effect = KeyError
 
-        self.assertRaises(KeyError, cls.request_vif, m_driver, pod, project_id,
-                          subnets, security_groups)
+        resp = cls.request_vif(m_driver, pod, project_id, subnets,
+                               security_groups)
+        self.assertIsNone(resp)
 
     @mock.patch('time.time', return_value=50)
     @ddt.data((neutron_vif.NeutronPodVIFDriver),

kuryr_kubernetes/utils.py

@@ -271,14 +271,22 @@ def set_lbaas_spec(service, lbaas_spec):
     try:
         k8s.annotate(ep_link,
                      {constants.K8S_ANNOTATION_LBAAS_SPEC: annotation})
-    except exceptions.K8sResourceNotFound:
+    except exceptions.K8sResourceNotFound as ex:
+        LOG.debug("Failed to annotate svc: %s", ex)
         raise exceptions.ResourceNotReady(ep_link)
     except exceptions.K8sClientException:
         LOG.debug("Failed to annotate endpoint %r", ep_link)
         raise
-    k8s.annotate(svc_link,
-                 {constants.K8S_ANNOTATION_LBAAS_SPEC: annotation},
-                 resource_version=service['metadata']['resourceVersion'])
+    try:
+        k8s.annotate(svc_link,
+                     {constants.K8S_ANNOTATION_LBAAS_SPEC: annotation},
+                     resource_version=service['metadata']['resourceVersion'])
+    except exceptions.K8sResourceNotFound as ex:
+        LOG.debug("Failed to annotate svc: %s", ex)
+        raise exceptions.ResourceNotReady(svc_link)
+    except exceptions.K8sClientException:
+        LOG.exception("Failed to annotate svc: %r", svc_link)
+        raise
 
 
 def get_lbaas_state(endpoint):