starlingx
/
nfv
Code Issues Proposed changes
Browse Source

Add retry capability to host unlock during upgrade 

During the unlock of a host as part of an upgrade, the
unlock can be rejected.

This change introduces a retry mechanism for the unlock.

Allow up to 5 retries with 2 minutes between attempts.

Partial-Bug: 1914836
Signed-off-by: albailey <Al.Bailey@windriver.com>
Change-Id: Ic121e1a993c80e2fae32806181342c1e5ea8e688
changes/10/780310/5
albailey 1 year ago
parent
a6d46eabef
commit
19b3a3f385
3 changed files with 275 additions and 163 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 180
      nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py
  2. 16
      nfv/nfv-vim/nfv_vim/strategy/_strategy.py
  3. 242
      nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py

180
nfv/nfv-tests/nfv_unit_tests/tests/test_sw_upgrade_strategy.py
Unescape View File

@ -22,10 +22,27 @@ from nfv_vim.nfvi.objects.v1 import UPGRADE_STATE
from . import sw_update_testcase # noqa: H304
@mock.patch('nfv_vim.event_log._instance._event_issue', sw_update_testcase.fake_event_issue)
@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save', sw_update_testcase.fake_save)
@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer', sw_update_testcase.fake_timer)
@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled', sw_update_testcase.fake_nfvi_compute_plugin_disabled)
# utility method for the formatting of unlock-hosts stage as dict
# workers default to 5 retries with 120 second delay between attempts
# std controllers and storage have 0 retries
def _unlock_hosts_stage_as_dict(host_names, retry_count=5, retry_delay=120):
return {
'name': 'unlock-hosts',
'entity_names': host_names,
'retry_count': retry_count,
'retry_delay': retry_delay,
'timeout': 1800,
}
@mock.patch('nfv_vim.event_log._instance._event_issue',
sw_update_testcase.fake_event_issue)
@mock.patch('nfv_vim.objects._sw_update.SwUpdate.save',
sw_update_testcase.fake_save)
@mock.patch('nfv_vim.objects._sw_update.timers.timers_create_timer',
sw_update_testcase.fake_timer)
@mock.patch('nfv_vim.nfvi.nfvi_compute_plugin_disabled',
sw_update_testcase.fake_nfvi_compute_plugin_disabled)
class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
def create_sw_upgrade_strategy(self,
@ -76,9 +93,10 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
"test_instance_1",
'compute-1')
self.create_instance_group('instance_group_1',
['test_instance_0', 'test_instance_1'],
[nfvi.objects.v1.INSTANCE_GROUP_POLICY.ANTI_AFFINITY])
self.create_instance_group(
'instance_group_1',
['test_instance_0', 'test_instance_1'],
[nfvi.objects.v1.INSTANCE_GROUP_POLICY.ANTI_AFFINITY])
worker_hosts = []
for host in self._host_table.values():
@ -162,8 +180,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-2', 'compute-3']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-2', 'compute-3']},
{'name': 'unlock-hosts',
'entity_names': ['compute-2', 'compute-3']},
_unlock_hosts_stage_as_dict(['compute-2', 'compute-3']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -179,8 +196,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-0']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-0']},
{'name': 'unlock-hosts',
'entity_names': ['compute-0']},
_unlock_hosts_stage_as_dict(['compute-0']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -196,8 +212,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-1']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-1']},
{'name': 'unlock-hosts',
'entity_names': ['compute-1']},
_unlock_hosts_stage_as_dict(['compute-1']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -268,8 +283,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-1', 'compute-5']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-1', 'compute-5']},
{'name': 'unlock-hosts',
'entity_names': ['compute-1', 'compute-5']},
_unlock_hosts_stage_as_dict(['compute-1', 'compute-5']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -287,8 +301,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-0', 'compute-2', 'compute-3']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-0', 'compute-2', 'compute-3']},
{'name': 'unlock-hosts',
'entity_names': ['compute-0', 'compute-2', 'compute-3']},
_unlock_hosts_stage_as_dict(
['compute-0', 'compute-2', 'compute-3']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -306,8 +320,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-4', 'compute-6', 'compute-7']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-4', 'compute-6', 'compute-7']},
{'name': 'unlock-hosts',
'entity_names': ['compute-4', 'compute-6', 'compute-7']},
_unlock_hosts_stage_as_dict(
['compute-4', 'compute-6', 'compute-7']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -324,8 +338,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-8', 'compute-9']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-8', 'compute-9']},
{'name': 'unlock-hosts',
'entity_names': ['compute-8', 'compute-9']},
_unlock_hosts_stage_as_dict(
['compute-8', 'compute-9']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -399,8 +413,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'unlock-hosts',
'entity_names': ['controller-0']},
_unlock_hosts_stage_as_dict(['controller-0']),
{'name': 'wait-data-sync',
'timeout': 14400}
]
@ -418,8 +431,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-1']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-1']},
{'name': 'unlock-hosts',
'entity_names': ['controller-1']},
_unlock_hosts_stage_as_dict(['controller-1']),
{'name': 'wait-data-sync',
'timeout': 14400}
]
@ -432,8 +444,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-1']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-1']},
{'name': 'unlock-hosts',
'entity_names': ['compute-1']},
_unlock_hosts_stage_as_dict(['compute-1']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -451,8 +462,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-0', 'compute-2', 'compute-3']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-0', 'compute-2', 'compute-3']},
{'name': 'unlock-hosts',
'entity_names': ['compute-0', 'compute-2', 'compute-3']},
_unlock_hosts_stage_as_dict(
['compute-0', 'compute-2', 'compute-3']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -468,8 +479,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-4']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-4']},
{'name': 'unlock-hosts',
'entity_names': ['compute-4']},
_unlock_hosts_stage_as_dict(['compute-4']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -558,8 +568,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': stage_hosts[0]},
{'name': 'upgrade-hosts',
'entity_names': stage_hosts[0]},
{'name': 'unlock-hosts',
'entity_names': stage_hosts[0]},
_unlock_hosts_stage_as_dict(stage_hosts[0]),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -580,8 +589,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': stage_hosts[x]},
{'name': 'upgrade-hosts',
'entity_names': stage_hosts[x]},
{'name': 'unlock-hosts',
'entity_names': stage_hosts[x]},
_unlock_hosts_stage_as_dict(stage_hosts[x]),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -644,8 +652,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-2']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-2']},
{'name': 'unlock-hosts',
'entity_names': ['compute-2']},
_unlock_hosts_stage_as_dict(['compute-2']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -658,8 +665,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-3']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-3']},
{'name': 'unlock-hosts',
'entity_names': ['compute-3']},
_unlock_hosts_stage_as_dict(['compute-3']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -674,8 +680,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-0']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-0']},
{'name': 'unlock-hosts',
'entity_names': ['compute-0']},
_unlock_hosts_stage_as_dict(['compute-0']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -690,8 +695,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-1']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-1']},
{'name': 'unlock-hosts',
'entity_names': ['compute-1']},
_unlock_hosts_stage_as_dict(['compute-1']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -745,8 +749,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-0']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-0']},
{'name': 'unlock-hosts',
'entity_names': ['compute-0']},
_unlock_hosts_stage_as_dict(['compute-0']),
{'name': 'system-stabilize'}
]
},
@ -758,8 +761,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-1']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-1']},
{'name': 'unlock-hosts',
'entity_names': ['compute-1']},
_unlock_hosts_stage_as_dict(['compute-1']),
{'name': 'system-stabilize'}
]
},
@ -771,8 +773,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-2']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-2']},
{'name': 'unlock-hosts',
'entity_names': ['compute-2']},
_unlock_hosts_stage_as_dict(['compute-2']),
{'name': 'system-stabilize'}
]
},
@ -784,8 +785,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-3']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-3']},
{'name': 'unlock-hosts',
'entity_names': ['compute-3']},
_unlock_hosts_stage_as_dict(['compute-3']),
{'name': 'system-stabilize'}
]
}
@ -936,8 +936,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-0']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-0']},
{'name': 'unlock-hosts',
'entity_names': ['storage-0']},
_unlock_hosts_stage_as_dict(['storage-0'], retry_count=0),
{'name': 'wait-data-sync',
'timeout': 7200}
]
@ -950,8 +949,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-1', 'storage-2']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-1', 'storage-2']},
{'name': 'unlock-hosts',
'entity_names': ['storage-1', 'storage-2']},
_unlock_hosts_stage_as_dict(['storage-1', 'storage-2'],
retry_count=0),
{'name': 'wait-data-sync',
'timeout': 7200}
]
@ -964,8 +963,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-3']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-3']},
{'name': 'unlock-hosts',
'entity_names': ['storage-3']},
_unlock_hosts_stage_as_dict(['storage-3'], retry_count=0),
{'name': 'wait-data-sync',
'timeout': 7200}
]
@ -1022,8 +1020,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-0']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-0']},
{'name': 'unlock-hosts',
'entity_names': ['storage-0']},
_unlock_hosts_stage_as_dict(['storage-0'], retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 7200}
@ -1037,8 +1034,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-1']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-1']},
{'name': 'unlock-hosts',
'entity_names': ['storage-1']},
_unlock_hosts_stage_as_dict(['storage-1'], retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 7200}
@ -1052,8 +1048,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-2']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-2']},
{'name': 'unlock-hosts',
'entity_names': ['storage-2']},
_unlock_hosts_stage_as_dict(['storage-2'], retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 7200}
@ -1067,8 +1062,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-3']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-3']},
{'name': 'unlock-hosts',
'entity_names': ['storage-3']},
_unlock_hosts_stage_as_dict(['storage-3'], retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 7200}
@ -1116,8 +1110,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'unlock-hosts',
'entity_names': ['controller-0']},
_unlock_hosts_stage_as_dict(['controller-0'],
retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1164,8 +1158,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-1']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-1']},
{'name': 'unlock-hosts',
'entity_names': ['controller-1']},
_unlock_hosts_stage_as_dict(['controller-1'],
retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1181,8 +1175,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'unlock-hosts',
'entity_names': ['controller-0']},
_unlock_hosts_stage_as_dict(['controller-0'],
retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1253,8 +1247,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'unlock-hosts',
'entity_names': ['controller-0']},
_unlock_hosts_stage_as_dict(['controller-0']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1270,8 +1263,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-1']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-1']},
{'name': 'unlock-hosts',
'entity_names': ['controller-1']},
_unlock_hosts_stage_as_dict(['controller-1']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1372,8 +1364,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-1']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-1']},
{'name': 'unlock-hosts',
'entity_names': ['controller-1']},
_unlock_hosts_stage_as_dict(['controller-1']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1387,8 +1378,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'unlock-hosts',
'entity_names': ['controller-0']},
_unlock_hosts_stage_as_dict(['controller-0']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1402,8 +1392,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-0']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-0']},
{'name': 'unlock-hosts',
'entity_names': ['storage-0']},
_unlock_hosts_stage_as_dict(['storage-0']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 7200}
@ -1417,8 +1406,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['storage-1']},
{'name': 'upgrade-hosts',
'entity_names': ['storage-1']},
{'name': 'unlock-hosts',
'entity_names': ['storage-1']},
_unlock_hosts_stage_as_dict(['storage-1']),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 7200}
@ -1432,8 +1420,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-1']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-1']},
{'name': 'unlock-hosts',
'entity_names': ['compute-1']},
_unlock_hosts_stage_as_dict(['compute-1']),
{'name': 'system-stabilize',
'timeout': 60}
]
@ -1448,8 +1435,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-0']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-0']},
{'name': 'unlock-hosts',
'entity_names': ['compute-0']},
_unlock_hosts_stage_as_dict(['compute-0']),
{'name': 'system-stabilize',
'timeout': 60}
]
@ -1516,8 +1502,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-1']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-1']},
{'name': 'unlock-hosts',
'entity_names': ['controller-1']},
_unlock_hosts_stage_as_dict(['controller-1'],
retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1533,8 +1519,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['controller-0']},
{'name': 'upgrade-hosts',
'entity_names': ['controller-0']},
{'name': 'unlock-hosts',
'entity_names': ['controller-0']},
_unlock_hosts_stage_as_dict(['controller-0'],
retry_count=0),
{'name': 'wait-data-sync',
'ignore_alarms': ['900.005', '900.201', '750.006'],
'timeout': 14400}
@ -1548,8 +1534,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-1']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-1']},
{'name': 'unlock-hosts',
'entity_names': ['compute-1']},
_unlock_hosts_stage_as_dict(['compute-1']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -1564,8 +1549,7 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
'entity_names': ['compute-0']},
{'name': 'upgrade-hosts',
'entity_names': ['compute-0']},
{'name': 'unlock-hosts',
'entity_names': ['compute-0']},
_unlock_hosts_stage_as_dict(['compute-0']),
{'name': 'wait-alarms-clear',
'timeout': 600}
]
@ -1794,7 +1778,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
expected_results = {
'total_stages': 0,
'result': 'failed',
'result_reason': 'all controller hosts must be unlocked-enabled-available'
'result_reason':
'all controller hosts must be unlocked-enabled-available'
}
sw_update_testcase.validate_phase(build_phase, expected_results)
@ -1840,7 +1825,8 @@ class TestSwUpgradeStrategy(sw_update_testcase.SwUpdateStrategyTestCase):
expected_results = {
'total_stages': 0,
'result': 'failed',
'result_reason': 'all worker hosts must be unlocked-enabled-available'
'result_reason':
'all worker hosts must be unlocked-enabled-available'
}
sw_update_testcase.validate_phase(build_phase, expected_results)

16
nfv/nfv-vim/nfv_vim/strategy/_strategy.py
Unescape View File

@ -1321,6 +1321,7 @@ class SwUpgradeStrategy(SwUpdateStrategy):
True, ignore_alarms=self._ignore_alarms))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# Note: standard controllers do not need the same retry as AIO
stage.add_step(strategy.UnlockHostsStep(host_list))
# Allow up to four hours for controller disks to synchronize
stage.add_step(strategy.WaitDataSyncStep(
@ -1341,6 +1342,7 @@ class SwUpgradeStrategy(SwUpdateStrategy):
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# Note: standard controllers do not need the same retry as AIO
stage.add_step(strategy.UnlockHostsStep(host_list))
# Allow up to four hours for controller disks to synchronize
stage.add_step(strategy.WaitDataSyncStep(
@ -1390,7 +1392,9 @@ class SwUpgradeStrategy(SwUpdateStrategy):
True, ignore_alarms=self._ignore_alarms))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
# storage hosts do not need the same retry logic as AIO
stage.add_step(strategy.UnlockHostsStep(host_list))
# After storage node(s) are unlocked, we need extra time to
# allow the OSDs to go back in sync and the storage related
# alarms to clear. We no longer wipe the OSD disks when upgrading
@ -1440,7 +1444,11 @@ class SwUpgradeStrategy(SwUpdateStrategy):
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
stage.add_step(strategy.UnlockHostsStep(host_list))
# During an upgrade, unlock may need to retry. Bug details:
# https://bugs.launchpad.net/starlingx/+bug/1914836
stage.add_step(strategy.UnlockHostsStep(
host_list,
retry_count=strategy.UnlockHostsStep.MAX_RETRIES))
if HOST_PERSONALITY.CONTROLLER in host_list[0].personality:
# AIO Controller hosts will undergo WaitDataSyncStep step
# Allow up to four hours for controller disks to synchronize
@ -1487,7 +1495,11 @@ class SwUpgradeStrategy(SwUpdateStrategy):
stage.add_step(strategy.SwactHostsStep(host_list))
stage.add_step(strategy.LockHostsStep(host_list))
stage.add_step(strategy.UpgradeHostsStep(host_list))
stage.add_step(strategy.UnlockHostsStep(host_list))
# During an upgrade, unlock may need to retry. Bug details:
# https://bugs.launchpad.net/starlingx/+bug/1914836
stage.add_step(strategy.UnlockHostsStep(
host_list,
retry_count=strategy.UnlockHostsStep.MAX_RETRIES))
if HOST_PERSONALITY.CONTROLLER in host_list[0].personality:
# AIO Controller hosts will undergo WaitDataSyncStep step
# Allow up to four hours for controller disks to synchronize

242
nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py
Unescape View File

@ -68,13 +68,44 @@ class StrategyStepNames(Constants):
STRATEGY_STEP_NAME = StrategyStepNames()
class UnlockHostsStep(strategy.StrategyStep):
"""
Unlock Hosts - Strategy Step
"""
def __init__(self, hosts):
super(UnlockHostsStep, self).__init__(
STRATEGY_STEP_NAME.UNLOCK_HOSTS, timeout_in_secs=1800)
class AbstractStrategyStep(strategy.StrategyStep):
"""An abstract base class for strategy steps"""
def __init__(self, step_name, timeout_in_secs):
super(AbstractStrategyStep, self).__init__(
step_name,
timeout_in_secs=timeout_in_secs)
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(AbstractStrategyStep, self).from_dict(data)
return self
def as_dict(self):
"""
Represent the step as a dictionary
"""
data = super(AbstractStrategyStep, self).as_dict()
# Next 3 lines are required for all strategy steps and may be
# overridden by subclass in some cases
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class AbstractHostsStrategyStep(AbstractStrategyStep):
"""An abstract base class for strategy steps performed on list of hosts"""
def __init__(self,
step_name,
hosts,
timeout_in_secs=1800):
super(AbstractHostsStrategyStep, self).__init__(
step_name,
timeout_in_secs=timeout_in_secs)
self._hosts = hosts
self._host_names = list()
self._host_uuids = list()
@ -82,6 +113,108 @@ class UnlockHostsStep(strategy.StrategyStep):
self._host_names.append(host.name)
self._host_uuids.append(host.uuid)
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(AbstractHostsStrategyStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
return self
def as_dict(self):
"""
Represent the step as a dictionary
"""
data = super(AbstractHostsStrategyStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class UnlockHostsStep(AbstractHostsStrategyStep):
"""
Unlock Hosts - Strategy Step
"""
# During an upgrade, an unlock may need to be retried several times
# https://bugs.launchpad.net/starlingx/+bug/1914836
MAX_RETRIES = 5
RETRY_DELAY = 120
def __init__(self, hosts, retry_count=0, retry_delay=RETRY_DELAY):
"""
hosts - the list of hosts to be unlocked
retry_count - the number of times to retry per host if unlock fails
retry_delay - the amount of time to delay before retrying unlock
"""
super(UnlockHostsStep, self).__init__(STRATEGY_STEP_NAME.UNLOCK_HOSTS,
hosts,
timeout_in_secs=1800)
# step_name, hosts, timeout are serialized by parent classes
# retry_count and retry_delay must be serialized in from_dict/as_dict
self._retry_count = retry_count
self._retry_delay = retry_delay
# Do not persist: _retries, _wait_time _retrying
self._retries = dict()
for host_name in self._host_names:
self._retries[host_name] = retry_count
self._wait_time = 0
self._retry_requested = False
def from_dict(self, data):
"""
Returns unlock hosts step object initialized using the given dictionary
"""
super(UnlockHostsStep, self).from_dict(data)
# deserialize retry_delay
self._retry_count = data['retry_count']
self._retry_delay = data['retry_delay']
# Do not deserialize _retries, _wait_time and _retrying
self._wait_time = 0
self._retry_requested = False
self._retries = dict()
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._retries[host_name] = self._retry_count
return self
def as_dict(self):
"""
Represent the unlock hosts step as a dictionary
"""
data = super(UnlockHostsStep, self).as_dict()
# serialize retries
data['retry_count'] = self._retry_count
# serialize retry_delay
data['retry_delay'] = self._retry_delay
# Do not serialize _retries, _wait_time and _retrying
return data
def _get_hosts_to_retry(self):
hosts = []
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is None:
continue
if host.is_locked() and self._retries[host_name] > 0:
self._retries[host_name] = self._retries[host_name] - 1
hosts.append(host_name)
return hosts
def _total_hosts_unlocked_enabled(self):
"""
Returns the number of hosts that are unlocked and enabled
@ -98,6 +231,16 @@ class UnlockHostsStep(strategy.StrategyStep):
return total_hosts_enabled
def _trigger_retry(self, host_name):
DLOG.info("Step (%s) retry due to failure for (%s)." % (self._name,
host_name))
# set the retry trigger
self._retry_requested = True
# reset the retry "wait" delay
self._wait_time = timers.get_monotonic_timestamp_in_ms()
# decrement the number of allowed retries for the validated host
self._retries[host_name] = self._retries[host_name] - 1
def apply(self):
"""
Unlock all hosts
@ -122,9 +265,12 @@ class UnlockHostsStep(strategy.StrategyStep):
"""
Handle Host events
"""
from nfv_vim import directors
DLOG.debug("Step (%s) handle event (%s)." % (self._name, event))
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED, STRATEGY_EVENT.HOST_AUDIT]:
if event in [STRATEGY_EVENT.HOST_STATE_CHANGED,
STRATEGY_EVENT.HOST_AUDIT]:
total_hosts_enabled = self._total_hosts_unlocked_enabled()
if -1 == total_hosts_enabled:
@ -137,41 +283,36 @@ class UnlockHostsStep(strategy.StrategyStep):
self.stage.step_complete(result, '')
return True
# See if we have requested a retry and are not currently retrying
if self._retry_requested:
now_ms = timers.get_monotonic_timestamp_in_ms()
secs_expired = (now_ms - self._wait_time) / 1000
if self._retry_delay <= secs_expired:
self._retry_requested = False
# re-issue unlock for all hosts.
# Hosts that are already unlocked or unlocking get skipped
host_director = directors.get_host_director()
operation = host_director.unlock_hosts(self._host_names)
if operation.is_failed():
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host unlock failed")
return True
elif event == STRATEGY_EVENT.HOST_UNLOCK_FAILED:
host = event_data
if host is not None and host.name in self._host_names:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host unlock failed")
if host.is_locked() and self._retries[host.name] > 0:
# if any unlock fails and we have retries, trigger it
# even if the last round of unlocks has not returned
self._trigger_retry(host.name)
else:
# if ANY unlock fails and we are out of retries, fail
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, "host unlock failed")
return True
return False
def from_dict(self, data):
"""
Returns the unlock hosts step object initialized using the given dictionary
"""
super(UnlockHostsStep, self).from_dict(data)
self._hosts = list()
self._host_uuids = list()
self._host_names = data['entity_names']
host_table = tables.tables_get_host_table()
for host_name in self._host_names:
host = host_table.get(host_name, None)
if host is not None:
self._hosts.append(host)
self._host_uuids.append(host.uuid)
return self
def as_dict(self):
"""
Represent the unlock hosts step as a dictionary
"""
data = super(UnlockHostsStep, self).as_dict()
data['entity_type'] = 'hosts'
data['entity_names'] = self._host_names
data['entity_uuids'] = self._host_uuids
return data
class LockHostsStep(strategy.StrategyStep):
"""
@ -2558,33 +2699,6 @@ class EnableHostServicesStep(strategy.StrategyStep):
return data
class AbstractStrategyStep(strategy.StrategyStep):
def __init__(self, step_name, timeout_in_secs):
super(AbstractStrategyStep, self).__init__(
step_name,
timeout_in_secs=timeout_in_secs)
def from_dict(self, data):
"""
Returns the step object initialized using the given dictionary
"""
super(AbstractStrategyStep, self).from_dict(data)
return self
def as_dict(self):
"""
Represent the step as a dictionary
"""
data = super(AbstractStrategyStep, self).as_dict()
# Next 3 lines are required for all strategy steps and may be
# overridden by subclass in some cases
data['entity_type'] = ''
data['entity_names'] = list()
data['entity_uuids'] = list()
return data
class ApplySwPatchesStep(AbstractStrategyStep):
"""
Apply Patches using patch API
@ -3053,7 +3167,7 @@ class KubeUpgradeNetworkingStep(AbstractKubeUpgradeStep):
class AbstractKubeHostUpgradeStep(AbstractKubeUpgradeStep):
"""Kube Upgrade Host - Abtsract Strategy Step
"""Kube Upgrade Host - Abstract Strategy Step
This operation issues a host command, which updates the kube upgrade object
"""

Write Preview
Loading…
Cancel
Save