diff --git a/octavia/controller/worker/v1/controller_worker.py b/octavia/controller/worker/v1/controller_worker.py index 7f4b23107e..3a27bbf7ae 100644 --- a/octavia/controller/worker/v1/controller_worker.py +++ b/octavia/controller/worker/v1/controller_worker.py @@ -457,12 +457,23 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine): log=LOG): delete_member_tf.run() + @tenacity.retry( + retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound), + wait=tenacity.wait_incrementing( + RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), + stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) def batch_update_members(self, old_member_ids, new_member_ids, updated_members): - old_members = [self._member_repo.get(db_apis.get_session(), id=mid) - for mid in old_member_ids] new_members = [self._member_repo.get(db_apis.get_session(), id=mid) for mid in new_member_ids] + # The API may not have commited all of the new member records yet. + # Make sure we retry looking them up. + if None in new_members or len(new_members) != len(new_member_ids): + LOG.warning('Failed to fetch one of the new members from DB. ' + 'Retrying for up to 60 seconds.') + raise db_exceptions.NoResultFound + old_members = [self._member_repo.get(db_apis.get_session(), id=mid) + for mid in old_member_ids] updated_members = [ (self._member_repo.get(db_apis.get_session(), id=m.get('id')), m) for m in updated_members] diff --git a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py index 7092fd55fc..4e488e119c 100644 --- a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py +++ b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py @@ -824,7 +824,8 @@ class TestControllerWorker(base.TestCase): mock_amp_repo_get): _flow_mock.reset_mock() - + mock_member_repo_get.side_effect = [None, _member_mock, + _member_mock, _member_mock] cw = controller_worker.ControllerWorker() cw.batch_update_members([9], [11], [MEMBER_UPDATE_DICT]) @@ -837,6 +838,7 @@ class TestControllerWorker(base.TestCase): constants.POOL: _pool_mock})) _flow_mock.run.assert_called_once_with() + self.assertEqual(4, mock_member_repo_get.call_count) @mock.patch('octavia.controller.worker.v1.flows.' 'pool_flows.PoolFlows.get_create_pool_flow',