x/cloudbase-init
Code Issues Proposed changes

Retry netlbfo team creation

Browse Source 
The WMI implementation for NETLBFO is sometimes flaky and does not fail
when a bond cannot be created.

Added 5 retries for the team creation, just in case the WMI call to
create the bond did not fail, but the bond was not created.

In this scenario, the _wait_for_nic method is called and it fails,
as it does not find the team nic and the retry is performed.

Updated the wait for team nic retry interval to 10 seconds,
as the team nic may be retrieved with WMI after a longer time.

Change-Id: I7622a4a30b867f335ed8f3288ca2fea830daf94b
This commit is contained in:
Adrian Vladu 2020-01-21 12:44:22 +02:00
parent 08641ae8b3
commit ab342fc493
2 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cloudbaseinit/tests/utils/windows/test_netlbfo.py
View File

@ -41,6 +41,7 @@ class NetLBFOTest(unittest.TestCase):
def tearDown(self):
self._module_patcher.stop()
@mock.patch('time.sleep')
@mock.patch(MODPATH + '.NetLBFOTeamManager._get_primary_adapter_name')
@mock.patch(MODPATH + '.NetLBFOTeamManager._create_team')
@mock.patch(MODPATH + '.NetLBFOTeamManager._add_team_member')
@ -49,10 +50,11 @@ class NetLBFOTest(unittest.TestCase):
@mock.patch(MODPATH + '.NetLBFOTeamManager.delete_team')
def _test_create_team(self, mock_delete_team, mock_wait_for_nic,
mock_set_primary_nic_vlan_id, mock_add_team_member,
mock_create_team, mock_get_primary_adapter_name,
mode_not_found=False, lb_algo_not_found=False,
mock_create_team, mock_primary_adapter_name,
mock_time_sleep, mode_not_found=False,
lb_algo_not_found=False,
add_team_member_fail=False):
mock_get_primary_adapter_name.return_value = mock.sentinel.pri_nic_name
mock_primary_adapter_name.return_value = mock.sentinel.pri_nic_name
mock_create_team.return_value = None
lacp_timer = network_model.BOND_LACP_RATE_FAST
@ -100,9 +102,6 @@ class NetLBFOTest(unittest.TestCase):
mock.sentinel.mac, mock.sentinel.pri_nic_name,
mock.sentinel.vlan_id, lacp_timer)
mock_add_team_member.assert_called_once_with(
conn, mock.sentinel.team_name, mock.sentinel.other_member)
if not add_team_member_fail:
mock_set_primary_nic_vlan_id.assert_called_once_with(
conn, mock.sentinel.team_name, mock.sentinel.vlan_id)
@ -111,8 +110,14 @@ class NetLBFOTest(unittest.TestCase):
2, 3, mock.sentinel.pri_nic_name, 1)
mock_wait_for_nic.assert_called_once_with(
mock_team_nic.Name)
mock_add_team_member.assert_called_once_with(
conn, mock.sentinel.team_name, mock.sentinel.other_member)
else:
mock_delete_team.assert_called_once_with(mock.sentinel.team_name)
mock_add_team_member.assert_called_with(
conn, mock.sentinel.team_name, mock.sentinel.other_member)
mock_delete_team.assert_called_with(mock.sentinel.team_name)
self.assertEqual(mock_add_team_member.call_count, 6)
self.assertEqual(mock_delete_team.call_count, 6)
def test_create_team(self):
self._test_create_team()

4
cloudbaseinit/utils/windows/netlbfo.py
View File

@ -132,6 +132,7 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
operation_options = {u'custom_options': custom_options}
team.put(operation_options=operation_options)
@retry_decorator.retry_decorator(max_retry_count=5)
def create_team(self, team_name, mode, load_balancing_algorithm,
members, mac_address, primary_nic_name=None,
primary_nic_vlan_id=None, lacp_timer=None):
@ -178,7 +179,8 @@ class NetLBFOTeamManager(network_team.BaseNetworkTeamManager):
raise ex
@staticmethod
@retry_decorator.retry_decorator(max_retry_count=10)
@retry_decorator.retry_decorator(max_retry_count=10,
max_sleep_time=10)
def _wait_for_nic(nic_name):
conn = wmi.WMI(moniker='//./root/cimv2')
if not conn.Win32_NetworkAdapter(NetConnectionID=nic_name):