From ab342fc4938fcc6d9c582deba52c418b567b5fc3 Mon Sep 17 00:00:00 2001 From: Adrian Vladu Date: Tue, 21 Jan 2020 12:44:22 +0200 Subject: [PATCH] Retry netlbfo team creation The WMI implementation for NETLBFO is sometimes flaky and does not fail when a bond cannot be created. Added 5 retries for the team creation, just in case the WMI call to create the bond did not fail, but the bond was not created. In this scenario, the _wait_for_nic method is called and it fails, as it does not find the team nic and the retry is performed. Updated the wait for team nic retry interval to 10 seconds, as the team nic may be retrieved with WMI after a longer time. Change-Id: I7622a4a30b867f335ed8f3288ca2fea830daf94b --- .../tests/utils/windows/test_netlbfo.py | 19 ++++++++++++------- cloudbaseinit/utils/windows/netlbfo.py | 4 +++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cloudbaseinit/tests/utils/windows/test_netlbfo.py b/cloudbaseinit/tests/utils/windows/test_netlbfo.py index dd65d32b..408e2b12 100644 --- a/cloudbaseinit/tests/utils/windows/test_netlbfo.py +++ b/cloudbaseinit/tests/utils/windows/test_netlbfo.py @@ -41,6 +41,7 @@ class NetLBFOTest(unittest.TestCase): def tearDown(self): self._module_patcher.stop() + @mock.patch('time.sleep') @mock.patch(MODPATH + '.NetLBFOTeamManager._get_primary_adapter_name') @mock.patch(MODPATH + '.NetLBFOTeamManager._create_team') @mock.patch(MODPATH + '.NetLBFOTeamManager._add_team_member') @@ -49,10 +50,11 @@ class NetLBFOTest(unittest.TestCase): @mock.patch(MODPATH + '.NetLBFOTeamManager.delete_team') def _test_create_team(self, mock_delete_team, mock_wait_for_nic, mock_set_primary_nic_vlan_id, mock_add_team_member, - mock_create_team, mock_get_primary_adapter_name, - mode_not_found=False, lb_algo_not_found=False, + mock_create_team, mock_primary_adapter_name, + mock_time_sleep, mode_not_found=False, + lb_algo_not_found=False, add_team_member_fail=False): - mock_get_primary_adapter_name.return_value = mock.sentinel.pri_nic_name + mock_primary_adapter_name.return_value = mock.sentinel.pri_nic_name mock_create_team.return_value = None lacp_timer = network_model.BOND_LACP_RATE_FAST @@ -100,9 +102,6 @@ class NetLBFOTest(unittest.TestCase): mock.sentinel.mac, mock.sentinel.pri_nic_name, mock.sentinel.vlan_id, lacp_timer) - mock_add_team_member.assert_called_once_with( - conn, mock.sentinel.team_name, mock.sentinel.other_member) - if not add_team_member_fail: mock_set_primary_nic_vlan_id.assert_called_once_with( conn, mock.sentinel.team_name, mock.sentinel.vlan_id) @@ -111,8 +110,14 @@ class NetLBFOTest(unittest.TestCase): 2, 3, mock.sentinel.pri_nic_name, 1) mock_wait_for_nic.assert_called_once_with( mock_team_nic.Name) + mock_add_team_member.assert_called_once_with( + conn, mock.sentinel.team_name, mock.sentinel.other_member) else: - mock_delete_team.assert_called_once_with(mock.sentinel.team_name) + mock_add_team_member.assert_called_with( + conn, mock.sentinel.team_name, mock.sentinel.other_member) + mock_delete_team.assert_called_with(mock.sentinel.team_name) + self.assertEqual(mock_add_team_member.call_count, 6) + self.assertEqual(mock_delete_team.call_count, 6) def test_create_team(self): self._test_create_team() diff --git a/cloudbaseinit/utils/windows/netlbfo.py b/cloudbaseinit/utils/windows/netlbfo.py index f225a016..685fee88 100644 --- a/cloudbaseinit/utils/windows/netlbfo.py +++ b/cloudbaseinit/utils/windows/netlbfo.py @@ -132,6 +132,7 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager): operation_options = {u'custom_options': custom_options} team.put(operation_options=operation_options) + @retry_decorator.retry_decorator(max_retry_count=5) def create_team(self, team_name, mode, load_balancing_algorithm, members, mac_address, primary_nic_name=None, primary_nic_vlan_id=None, lacp_timer=None): @@ -178,7 +179,8 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager): raise ex @staticmethod - @retry_decorator.retry_decorator(max_retry_count=10) + @retry_decorator.retry_decorator(max_retry_count=10, + max_sleep_time=10) def _wait_for_nic(nic_name): conn = wmi.WMI(moniker='//./root/cimv2') if not conn.Win32_NetworkAdapter(NetConnectionID=nic_name):