From 630a4e6a3cb60a8edeb4d8ad4a9cd7b31b42469a Mon Sep 17 00:00:00 2001 From: Michael Johnson Date: Thu, 4 Jun 2020 10:45:35 -0700 Subject: [PATCH] Fix batch member create for v1 amphora driver A previous patch[1] missed batch_member_update when adding database repository "get" method retries for new object creation actions. This patch fixes batch member create to retry the database get call when new members are being created via batch member update. This issue only impacts the v1 amphora driver as the v2 driver does not need to get these objects from the database. Story: 2007581 Task: 39503 [1] https://github.com/openstack/octavia/commit/48e85569f7e43e3dd5f09fd1ef4cb165526a92cd Change-Id: Ia3476ab7b24dc3fd6e29ff2abe6eb6bacd9908ed --- octavia/controller/worker/v1/controller_worker.py | 15 +++++++++++++-- .../worker/v1/test_controller_worker.py | 3 +++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/octavia/controller/worker/v1/controller_worker.py b/octavia/controller/worker/v1/controller_worker.py index a33a43c08e..3ecb336f98 100644 --- a/octavia/controller/worker/v1/controller_worker.py +++ b/octavia/controller/worker/v1/controller_worker.py @@ -483,12 +483,23 @@ class ControllerWorker(base_taskflow.BaseTaskFlowEngine): log=LOG): delete_member_tf.run() + @tenacity.retry( + retry=tenacity.retry_if_exception_type(db_exceptions.NoResultFound), + wait=tenacity.wait_incrementing( + RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX), + stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS)) def batch_update_members(self, old_member_ids, new_member_ids, updated_members): - old_members = [self._member_repo.get(db_apis.get_session(), id=mid) - for mid in old_member_ids] new_members = [self._member_repo.get(db_apis.get_session(), id=mid) for mid in new_member_ids] + # The API may not have commited all of the new member records yet. + # Make sure we retry looking them up. + if None in new_members or len(new_members) != len(new_member_ids): + LOG.warning('Failed to fetch one of the new members from DB. ' + 'Retrying for up to 60 seconds.') + raise db_exceptions.NoResultFound + old_members = [self._member_repo.get(db_apis.get_session(), id=mid) + for mid in old_member_ids] updated_members = [ (self._member_repo.get(db_apis.get_session(), id=m.get('id')), m) for m in updated_members] diff --git a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py index ec872ffa1b..6d79ae54ef 100644 --- a/octavia/tests/unit/controller/worker/v1/test_controller_worker.py +++ b/octavia/tests/unit/controller/worker/v1/test_controller_worker.py @@ -884,6 +884,8 @@ class TestControllerWorker(base.TestCase): mock_amp_repo_get): _flow_mock.reset_mock() + mock_member_repo_get.side_effect = [None, _member_mock, + _member_mock, _member_mock] mock_get_az_metadata_dict.return_value = {} cw = controller_worker.ControllerWorker() cw.batch_update_members([9], [11], [MEMBER_UPDATE_DICT]) @@ -898,6 +900,7 @@ class TestControllerWorker(base.TestCase): constants.AVAILABILITY_ZONE: {}})) _flow_mock.run.assert_called_once_with() + self.assertEqual(4, mock_member_repo_get.call_count) @mock.patch('octavia.controller.worker.v1.flows.' 'pool_flows.PoolFlows.get_create_pool_flow',