From 83f4525fec794b74dda147c1fc0136ebdedc1d36 Mon Sep 17 00:00:00 2001 From: Kobi Samoray Date: Tue, 20 Apr 2021 18:20:02 +0300 Subject: [PATCH] Octavia: retry update status on exception When the Octavia driver agent status update results with an exception, the driver should retry. Change-Id: Ia42e99a311b872ed87efbaaf93eb375a7620f740 (cherry picked from commit 36768b21c4fb00c8b2c492c59601ff7061240bf7) --- .../services/lbaas/octavia/octavia_driver.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/vmware_nsx/services/lbaas/octavia/octavia_driver.py b/vmware_nsx/services/lbaas/octavia/octavia_driver.py index d7815f25bc..63cfa9cf02 100644 --- a/vmware_nsx/services/lbaas/octavia/octavia_driver.py +++ b/vmware_nsx/services/lbaas/octavia/octavia_driver.py @@ -23,6 +23,7 @@ from oslo_log import helpers as log_helpers from oslo_log import log as logging import oslo_messaging as messaging from oslo_messaging.rpc import dispatcher +import tenacity from octavia.api.drivers import utils as oct_utils from octavia.db import api as db_apis @@ -42,6 +43,12 @@ TRANSPORT = None RPC_SERVER = None +def _log_before_retry(retry_state): + if retry_state.attempt_number > 0: + LOG.warning("Retrying the call to _update_loadbalancer_status due to " + "timeout") + + def get_transport(): global TRANSPORT if not TRANSPORT: @@ -670,12 +677,21 @@ class NSXOctaviaDriverEndpoint(driver_lib.DriverLibrary): self._removed_not_in_db(status, 'pools', 'pool') try: - return super(NSXOctaviaDriverEndpoint, - self).update_loadbalancer_status(status) + return self._update_loadbalancer_status(status) except exceptions.UpdateStatusError as e: LOG.error("Failed to update Octavia loadbalancer status. " "Status %s, Error %s", status, e.fault_string) + @tenacity.retry( + stop=tenacity.stop_after_attempt(30), reraise=True, + wait=tenacity.wait_exponential(multiplier=1, min=1, max=5), + before=_log_before_retry, + retry=tenacity.retry_if_exception_type( + exceptions.UpdateStatusError)) + def _update_loadbalancer_status(self, status): + super(NSXOctaviaDriverEndpoint, + self).update_loadbalancer_status(status) + @log_helpers.log_method_call def update_listener_statistics(self, ctxt, statistics): # refresh the driver lib session