diff --git a/octavia/controller/worker/v1/controller_worker.py b/octavia/controller/worker/v1/controller_worker.py index a33a43c08e..3ecb336f98 100644 --- a/octavia/controller/worker/v1/controller_worker.py +++ b/octavia/controller/worker/v1/controller_worker.py @@ -483,12 +483,23 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine): log=LOG): delete_member_tf.run() + @tenacity.retry( + retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound), + wait=tenacity.wait_incrementing( + RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), + stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) def batch_update_members(self, old_member_ids, new_member_ids, updated_members): - old_members = [self._member_repo.get(db_apis.get_session(), id=mid) - for mid in old_member_ids] new_members = [self._member_repo.get(db_apis.get_session(), id=mid) for mid in new_member_ids] + # The API may not have commited all of the new member records yet. + # Make sure we retry looking them up. + if None in new_members or len(new_members) != len(new_member_ids): + LOG.warning('Failed to fetch one of the new members from DB. ' + 'Retrying for up to 60 seconds.') + raise db_exceptions.NoResultFound + old_members = [self._member_repo.get(db_apis.get_session(), id=mid) + for mid in old_member_ids] updated_members = [ (self._member_repo.get(db_apis.get_session(), id=m.get('id')), m) for m in updated_members] diff --git a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py index ec872ffa1b..6d79ae54ef 100644 --- a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py +++ b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py @@ -884,6 +884,8 @@ class TestControllerWorker(base.TestCase): mock_amp_repo_get): _flow_mock.reset_mock() + mock_member_repo_get.side_effect = [None, _member_mock, + _member_mock, _member_mock] mock_get_az_metadata_dict.return_value = {} cw = controller_worker.ControllerWorker() cw.batch_update_members([9], [11], [MEMBER_UPDATE_DICT]) @@ -898,6 +900,7 @@ class TestControllerWorker(base.TestCase): constants.AVAILABILITY_ZONE: {}})) _flow_mock.run.assert_called_once_with() + self.assertEqual(4, mock_member_repo_get.call_count) @mock.patch('octavia.controller.worker.v1.flows.' 'pool_flows.PoolFlows.get_create_pool_flow',