openstack
/
octavia
Code Issues Proposed changes

Fix health manager messages about failed failovers 

The health manager displayed some incorrect statistics about failed
failovers, they were marked as "successfully completed failovers".
The exception from the failover_amphora function was not reraised when
called from health-manager, now the exception is reraised, allowing the
health-manager to compute correct statistics.

Story 2008636
Task 41876

Change-Id: Icb1430a3709566796e39fbfa962c2541c504b9ae
This commit is contained in:
Gregory Thiemonge 2021-02-18 12:13:50 +01:00
parent 38887c6f0f
commit 3f6fb8af22
5 changed files with 58 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
octavia/controller/healthmanager/health_manager.py
View File

@ -136,7 +136,7 @@ class HealthManager(object):
LOG.info("Stale amphora's id is: %s", amp_health.amphora_id)
fut = self.executor.submit(
self.cw.failover_amphora, amp_health.amphora_id)
self.cw.failover_amphora, amp_health.amphora_id, reraise=True)
fut.add_done_callback(
functools.partial(update_stats_on_done, stats)
)

5
octavia/controller/worker/v1/controller_worker.py
View File

@ -874,13 +874,14 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine):
log=LOG):
update_l7rule_tf.run()
def failover_amphora(self, amphora_id):
def failover_amphora(self, amphora_id, reraise=False):
"""Perform failover operations for an amphora.
Note: This expects the load balancer to already be in
provisioning_status=PENDING_UPDATE state.
:param amphora_id: ID for amphora to failover
:param reraise: If enabled reraise any caught exception
:returns: None
:raises octavia.common.exceptions.NotFound: The referenced amphora was
not found
@ -968,7 +969,7 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine):
"role": amphora.role})
except Exception as e:
with excutils.save_and_reraise_exception(reraise=False):
with excutils.save_and_reraise_exception(reraise=reraise):
LOG.exception("Amphora %s failover exception: %s",
amphora_id, str(e))
self._amphora_repo.update(db_apis.get_session(),

5
octavia/controller/worker/v2/controller_worker.py
View File

@ -865,13 +865,14 @@ class ControllerWorker(object):
flow_utils.get_update_l7rule_flow,
store=store)
def failover_amphora(self, amphora_id):
def failover_amphora(self, amphora_id, reraise=False):
"""Perform failover operations for an amphora.
Note: This expects the load balancer to already be in
provisioning_status=PENDING_UPDATE state.
:param amphora_id: ID for amphora to failover
:param reraise: If enabled reraise any caught exception
:returns: None
:raises octavia.common.exceptions.NotFound: The referenced amphora was
not found
@ -960,7 +961,7 @@ class ControllerWorker(object):
"role": amphora.role})
except Exception as e:
with excutils.save_and_reraise_exception(reraise=False):
with excutils.save_and_reraise_exception(reraise=reraise):
LOG.exception("Amphora %s failover exception: %s",
amphora_id, str(e))
self._amphora_repo.update(db_apis.get_session(),

26
octavia/tests/unit/controller/worker/v1/test_controller_worker.py
View File

@ -1559,6 +1559,32 @@ class TestControllerWorker(base.TestCase):
mock_update.assert_called_with(_db_session, LB_ID,
provisioning_status=constants.ERROR)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.update')
def test_failover_amp_flow_exception_reraise(self,
mock_update,
mock_api_get_session,
mock_dyn_log_listener,
mock_taskflow_load,
mock_pool_repo_get,
mock_member_repo_get,
mock_l7rule_repo_get,
mock_l7policy_repo_get,
mock_listener_repo_get,
mock_lb_repo_get,
mock_health_mon_repo_get,
mock_amp_repo_get):
mock_amphora = mock.MagicMock()
mock_amphora.id = AMP_ID
mock_amphora.load_balancer_id = LB_ID
mock_amp_repo_get.return_value = mock_amphora
mock_lb_repo_get.side_effect = TestException('boom')
cw = controller_worker.ControllerWorker()
self.assertRaises(TestException,
cw.failover_amphora,
AMP_ID, reraise=True)
@mock.patch('octavia.controller.worker.v1.flows.amphora_flows.'
'AmphoraFlows.get_failover_amphora_flow')
def test_failover_amp_no_lb(self,

25
octavia/tests/unit/controller/worker/v2/test_controller_worker.py
View File

@ -1613,6 +1613,31 @@ class TestControllerWorker(base.TestCase):
mock_update.assert_called_with(_db_session, LB_ID,
provisioning_status=constants.ERROR)
@mock.patch('octavia.db.repositories.LoadBalancerRepository.update')
def test_failover_amp_flow_exception_reraise(self,
mock_update,
mock_api_get_session,
mock_dyn_log_listener,
mock_taskflow_load,
mock_pool_repo_get,
mock_member_repo_get,
mock_l7rule_repo_get,
mock_l7policy_repo_get,
mock_listener_repo_get,
mock_lb_repo_get,
mock_health_mon_repo_get,
mock_amp_repo_get):
mock_amphora = mock.MagicMock()
mock_amphora.id = AMP_ID
mock_amphora.load_balancer_id = LB_ID
mock_amp_repo_get.return_value = mock_amphora
mock_lb_repo_get.side_effect = TestException('boom')
cw = controller_worker.ControllerWorker()
self.assertRaises(TestException,
cw.failover_amphora,
AMP_ID, reraise=True)
@mock.patch('octavia.controller.worker.v1.flows.amphora_flows.'
'AmphoraFlows.get_failover_amphora_flow')
def test_failover_amp_no_lb(self,