From 11ccc6611811cc1397ace28780415c93ffa8b467 Mon Sep 17 00:00:00 2001 From: Maysa Macedo Date: Thu, 21 Nov 2019 14:18:09 +0000 Subject: [PATCH] Prevent Kuryr restart due to connection issue In case the connection to Keystone is broken Kuryr controller is instantly restarted. This may result in leftovers loadbalancers as a Kubernetes service deletion can happen while the lbaas is still in PENDING_CREATE and if the controller is restarted, the deletion event will be gone. In this commit we give Kuryr a chance to retry the event before restarting. Change-Id: Ie2cacc86e745bbd7da7254f7adfd9b41370936ab Closes-bug: 1853460 --- kuryr_kubernetes/controller/handlers/pipeline.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kuryr_kubernetes/controller/handlers/pipeline.py b/kuryr_kubernetes/controller/handlers/pipeline.py index 3f0cd86d1..384be01f0 100644 --- a/kuryr_kubernetes/controller/handlers/pipeline.py +++ b/kuryr_kubernetes/controller/handlers/pipeline.py @@ -13,6 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +from requests import exceptions as requests_exc + +from keystoneauth1 import exceptions as key_exc + from kuryr_kubernetes import exceptions from kuryr_kubernetes.handlers import asynchronous as h_async from kuryr_kubernetes.handlers import dispatch as h_dis @@ -55,7 +59,10 @@ class ControllerPipeline(h_dis.EventPipeline): def _wrap_consumer(self, consumer): # TODO(ivc): tune retry interval/timeout return h_log.LogExceptions(h_retry.Retry( - consumer, exceptions=exceptions.ResourceNotReady)) + consumer, exceptions=( + exceptions.ResourceNotReady, + key_exc.connection.ConnectFailure, + requests_exc.ConnectionError))) def _wrap_dispatcher(self, dispatcher): return h_log.LogExceptions(h_async.Async(dispatcher, self._tg,