openstack/tacker
Code Issues Proposed changes

Support Anti-Affinity rules in AZ reselection

Browse Source 
This patch enhances availability zone reselection to select
zones based on Anti-Affinity rules. The zones subject to the
Anti-Affinity rules in availability zone reselection are
selected if there are no more zones available.

Implements: blueprint enhance-placement
Change-Id: Ie858156d9db00e9962b077c49d0dfe1f034d55fe
This commit is contained in:
Ken Fujimoto
2023-04-26 08:58:02 +00:00
parent f0da0ec589
commit 62a01ce2c0
2 changed files with 192 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
tacker/sol_refactored/infra_drivers/openstack/openstack.py
View File

@@ -105,15 +105,17 @@ class Openstack(object):
try:
stack_id = heat_client.create_stack(fields)
except sol_ex.StackOperationFailed as ex:
anti_rules = vnfd.get_anti_affinity_targets(req.flavourId)
self._update_stack_retry(heat_client, fields, inst, None,
ex, vim_info, vdu_ids)
ex, vim_info, vdu_ids, anti_rules)
stack_id = heat_client.get_stack_id(stack_name)
else:
try:
heat_client.update_stack(f'{stack_name}/{stack_id}', fields)
except sol_ex.StackOperationFailed as ex:
anti_rules = vnfd.get_anti_affinity_targets(req.flavourId)
self._update_stack_retry(heat_client, fields, inst, stack_id,
ex, vim_info, vdu_ids)
ex, vim_info, vdu_ids, anti_rules)
# make instantiated_vnf_info
self._make_instantiated_vnf_info(req, inst, grant_req, grant, vnfd,
@@ -201,8 +203,10 @@ class Openstack(object):
heat_client.update_stack(stack_name, fields)
except sol_ex.StackOperationFailed as ex:
if req.type == 'SCALE_OUT':
anti_rules = vnfd.get_anti_affinity_targets(
inst.instantiatedVnfInfo.flavourId)
self._update_stack_retry(heat_client, fields, inst, None, ex,
vim_info, vdu_ids)
vim_info, vdu_ids, anti_rules)
else:
raise ex
@@ -311,8 +315,10 @@ class Openstack(object):
stack_id = heat_client.create_stack(fields)
except sol_ex.StackOperationFailed as ex:
vdu_ids = self._get_vdu_id_from_grant_req(grant_req, inst)
anti_rules = vnfd.get_anti_affinity_targets(
inst.instantiatedVnfInfo.flavourId)
self._update_stack_retry(heat_client, fields, inst, None,
ex, vim_info, vdu_ids)
ex, vim_info, vdu_ids, anti_rules)
stack_id = heat_client.get_stack_id(stack_name)
else:
# mark unhealthy to target resources.
@@ -343,8 +349,10 @@ class Openstack(object):
heat_client.update_stack(stack_name, fields)
except sol_ex.StackOperationFailed as ex:
vdu_ids = self._get_vdu_id_from_grant_req(grant_req, inst)
anti_rules = vnfd.get_anti_affinity_targets(
inst.instantiatedVnfInfo.flavourId)
self._update_stack_retry(heat_client, fields, inst, None,
ex, vim_info, vdu_ids)
ex, vim_info, vdu_ids, anti_rules)
stack_id = inst.instantiatedVnfInfo.metadata['stack_id']
@@ -668,17 +676,14 @@ class Openstack(object):
heat_client, is_rollback=True)
def _update_stack_retry(self, heat_client, fields, inst, stack_id,
error_ex, vim_info, vdu_ids):
# NOTE: This method first selects a zone from unused zones
# and retries in case of failure due to zone resource constraints.
# If there are no unused zones, it selects from the used zones.
error_ex, vim_info, vdu_ids, anti_rules):
if not CONF.v2_vnfm.placement_fallback_best_effort:
# NOTE: If fallback_best_effort is False,
# AZ reselection is not executed.
raise error_ex
vdu_dict = fields['parameters']['nfv']['VDU']
failed_zone = self._check_and_get_failed_zone(
failed_vdu_id, failed_zone = self._check_and_get_failed_zone(
error_ex.detail, vdu_dict)
if failed_zone is None:
raise error_ex
@@ -686,25 +691,22 @@ class Openstack(object):
stack_name = heat_utils.get_stack_name(inst, stack_id)
nova_client = nova_utils.NovaClient(vim_info)
zones = nova_client.get_zone()
used_zones = {parameters.get('locationConstraints')
for parameters in vdu_dict.values()
if parameters.get('locationConstraints') is not None}
if (inst.obj_attr_is_set('instantiatedVnfInfo') and
inst.instantiatedVnfInfo.obj_attr_is_set('vnfcResourceInfo')):
used_zones |= {vnfc.metadata.get('zone') for vnfc
in inst.instantiatedVnfInfo.vnfcResourceInfo
if vnfc.metadata.get('zone') is not None}
failed_zones = set()
failed_zones.add(failed_zone)
available_zones = zones - used_zones
used_zones.discard(failed_zone)
retry_count = (CONF.v2_vnfm.placement_az_select_retry
if CONF.v2_vnfm.placement_az_select_retry
else len(zones))
while retry_count > 0:
if available_zones:
new_zone = available_zones.pop()
elif used_zones:
new_zone = used_zones.pop()
exclude_zones = self._get_exclude_zone(
inst, anti_rules, failed_vdu_id, vdu_ids, vdu_dict)
if zones - exclude_zones - failed_zones:
new_zone = list(zones - exclude_zones - failed_zones)[0]
elif exclude_zones - failed_zones:
# If there is no zone that matches the Anti-Affinity rules,
# selected zone does not comply with the Anti-Affinity
# rules.
new_zone = list(exclude_zones - failed_zones)[0]
else:
message = ("Availability Zone reselection failed. "
"No Availability Zone available.")
@@ -721,10 +723,11 @@ class Openstack(object):
heat_client.update_stack(stack_name, fields)
return
except sol_ex.StackOperationFailed as ex:
failed_zone = self._check_and_get_failed_zone(
failed_vdu_id, failed_zone = self._check_and_get_failed_zone(
ex.detail, vdu_dict)
if failed_zone is None:
raise ex
failed_zones.add(failed_zone)
retry_count -= 1
error_ex = ex
else:
@@ -734,14 +737,19 @@ class Openstack(object):
raise error_ex
def _check_and_get_failed_zone(self, ex_detail, vdu_dict):
if re.match(CONF.v2_vnfm.placement_az_resource_error, ex_detail):
match_result = re.search(r'resources\.((.*)-([0-9]+))', ex_detail)
if match_result is None:
LOG.warning("CONF v2_vnfm.placement_az_resource_error is "
"invalid. Please check.")
return None
vdu_id = match_result.group(1)
return vdu_dict.get(vdu_id, {}).get('locationConstraints')
if not re.match(CONF.v2_vnfm.placement_az_resource_error, ex_detail):
return None, None
match_result = re.search(r'resources\.((.*)-([0-9]+))', ex_detail)
if match_result is None:
LOG.warning(
"CONF v2_vnfm.placement_az_resource_error is invalid. "
"'{}' is not match. Please check it.".format(ex_detail))
return None, None
vdu_id = match_result.group(1)
return (vdu_id,
vdu_dict.get(vdu_id, {}).get('locationConstraints'))
def _get_vdu_id_from_fields(self, fields):
vdu_dict = fields['parameters']['nfv']['VDU']
@@ -756,6 +764,44 @@ class Openstack(object):
if vnfc.computeResource.resourceId in vnfc_res_ids}
return vdu_ids
def _get_anti_vdus(self, anti_rules, target_vdu):
anti_vdus = set()
for (targets, scope) in anti_rules:
if scope == 'zone' and target_vdu in targets:
if len(targets) == 1:
anti_vdus.add(target_vdu)
else:
anti_vdus |= {vdu for vdu in targets if vdu != target_vdu}
return anti_vdus
def _get_exclude_zone(self, inst, anti_rules, failed_vdu_id, vdu_ids,
vdu_dict):
# return zones which are used by VDUs (in inst and vdu_dict[vdu_ids])
# that are Anti-Affinity relation(anti_rules) of re-selection target
# VDU(failed_vdu_id).
def _get_vdu_from_vdu_with_idx(vdu_with_idx):
part = vdu_with_idx.rpartition('-')
if part[1] == '':
return None
return part[0]
target_vdu = _get_vdu_from_vdu_with_idx(failed_vdu_id)
anti_vdus = self._get_anti_vdus(anti_rules, target_vdu)
exclude_zones = {vdu_dict[vdu_id].get('locationConstraints')
for vdu_id in vdu_ids
if (_get_vdu_from_vdu_with_idx(vdu_id) in anti_vdus
and vdu_dict.get(vdu_id, {}).get(
'locationConstraints') is not None)}
if (inst.obj_attr_is_set('instantiatedVnfInfo') and
inst.instantiatedVnfInfo.obj_attr_is_set('vnfcResourceInfo')):
exclude_zones |= {vnfc.metadata.get('zone') for vnfc
in inst.instantiatedVnfInfo.vnfcResourceInfo
if (vnfc.vduId in anti_vdus and
vnfc.metadata.get('zone') is not None)}
return exclude_zones
def _make_hot(self, req, inst, grant_req, grant, vnfd, is_rollback=False):
if grant_req.operation == v2fields.LcmOperationType.INSTANTIATE:
flavour_id = req.flavourId

202
tacker/tests/unit/sol_refactored/infra_drivers/openstack/test_openstack.py
View File

@@ -3502,10 +3502,11 @@ _fields_example_scale = {
'VDU1-0': {
'computeFlavourId': 'm1.tiny',
'vcImageId': 'cirros-0.5.2-x86_64-disk',
'locationConstraints': 'az-2'
'locationConstraints': 'az-1'
},
'VDU2-0': {
'computeFlavourId': 'm1.tiny',
'locationConstraints': 'az-2'
},
'VDU2-VirtualStorage-0': {
'vcImageId': '0fea3414-93c0-46f5-b042-857be40e9fc7'
@@ -3525,9 +3526,41 @@ _fields_example_scale = {
}
}
_vdu_dict_example_get_exclude_zone = {
'VDU1-0': {
'computeFlavourId': 'm1.tiny',
'vcImageId': 'cirros-0.5.2-x86_64-disk',
'locationConstraints': 'az-1'
},
'VDU2-0': {
'computeFlavourId': 'm1.tiny',
'locationConstraints': 'az-2'
},
'VDU2-VirtualStorage-0': {
'vcImageId': '0fea3414-93c0-46f5-b042-857be40e9fc7'
},
'VDU1-1': {
'computeFlavourId': 'm1.tiny',
'vcImageId': 'cirros-0.5.2-x86_64-disk',
'locationConstraints': 'az-3'
},
'VDU1-2': {
'computeFlavourId': 'm1.tiny',
'vcImageId': 'cirros-0.5.2-x86_64-disk',
'locationConstraints': 'az-4'
}
}
_update_retry_instantiated_vnfinfo = {
"vnfcResourceInfo": [
{
"vduId": "VDU1",
"metadata": {
"zone": "az-1"
}
},
{
"vduId": "VDU2",
"metadata": {
"zone": "az-2"
}
@@ -4122,9 +4155,10 @@ class TestOpenstack(base.BaseTestCase):
error_ex = sol_ex.StackOperationFailed(sol_detail=sol_detail,
sol_title="stack failed")
ex = self.assertRaises(sol_ex.StackOperationFailed,
self.driver._update_stack_retry, mock.Mock(), mock.Mock(),
mock.Mock(), mock.Mock(), error_ex, mock.Mock(), mock.Mock())
ex = self.assertRaises(
sol_ex.StackOperationFailed, self.driver._update_stack_retry,
mock.Mock(), mock.Mock(), mock.Mock(), mock.Mock(), error_ex,
mock.Mock(), mock.Mock(), mock.Mock())
self.assertEqual(error_ex.detail, ex.detail)
mock_update_stack.assert_not_called()
@@ -4139,9 +4173,10 @@ class TestOpenstack(base.BaseTestCase):
error_ex = sol_ex.StackOperationFailed(sol_detail=sol_detail,
sol_title="stack failed")
ex = self.assertRaises(sol_ex.StackOperationFailed,
self.driver._update_stack_retry, mock.Mock(), fields_example,
mock.Mock(), mock.Mock(), error_ex, mock.Mock(), mock.Mock())
ex = self.assertRaises(
sol_ex.StackOperationFailed, self.driver._update_stack_retry,
mock.Mock(), fields_example, mock.Mock(), mock.Mock(), error_ex,
mock.Mock(), mock.Mock(), mock.Mock())
self.assertEqual(error_ex.detail, ex.detail)
mock_update_stack.assert_not_called()
@@ -4160,9 +4195,10 @@ class TestOpenstack(base.BaseTestCase):
"\"Message: No valid host was found. , Code: 500\"")
error_ex = sol_ex.StackOperationFailed(sol_detail=sol_detail,
sol_title="stack failed")
ex = self.assertRaises(sol_ex.StackOperationFailed,
self.driver._update_stack_retry, mock.Mock(), fields_example,
mock.Mock(), mock.Mock(), error_ex, mock.Mock(), mock.Mock())
ex = self.assertRaises(
sol_ex.StackOperationFailed, self.driver._update_stack_retry,
mock.Mock(), fields_example, mock.Mock(), mock.Mock(), error_ex,
mock.Mock(), mock.Mock(), mock.Mock())
self.assertEqual(error_ex.detail, ex.detail)
mock_update_stack.assert_not_called()
@@ -4178,11 +4214,14 @@ class TestOpenstack(base.BaseTestCase):
vim_info = objects.VimConnectionInfo.from_dict(
_vim_connection_info_example)
inst = objects.VnfInstanceV2(id=uuidutils.generate_uuid())
inst = objects.VnfInstanceV2(
id=uuidutils.generate_uuid()
)
fields_example = copy.deepcopy(_fields_example_instantiate)
heat_client = openstack.heat_utils.HeatClient(vim_info)
vdu_ids = {"VDU1-0", "VDU2-0", "VDU2-VirtualStorage-0"}
anti_rules = []
sol_detail = ("Resource CREATE failed: ResourceInError: resources."
"VDU1-0.resources.VDU1: Went to status ERROR due to "
@@ -4203,9 +4242,10 @@ class TestOpenstack(base.BaseTestCase):
mock_update_stack.side_effect = _retry
# execute
self.assertRaises(sol_ex.StackOperationFailed,
self.driver._update_stack_retry, heat_client, fields_example,
inst, STACK_ID, error_ex, vim_info, vdu_ids)
self.assertRaises(
sol_ex.StackOperationFailed, self.driver._update_stack_retry,
heat_client, fields_example, inst, STACK_ID, error_ex, vim_info,
vdu_ids, anti_rules)
self.assertEqual(len(mock_get_zone.return_value) - 1,
mock_update_stack.call_count)
@@ -4220,11 +4260,14 @@ class TestOpenstack(base.BaseTestCase):
vim_info = objects.VimConnectionInfo.from_dict(
_vim_connection_info_example)
inst = objects.VnfInstanceV2(id=uuidutils.generate_uuid())
inst = objects.VnfInstanceV2(
id=uuidutils.generate_uuid()
)
fields_example = copy.deepcopy(_fields_example_instantiate)
heat_client = openstack.heat_utils.HeatClient(vim_info)
vdu_ids = {"VDU1-0", "VDU2-0", "VDU2-VirtualStorage-0"}
anti_rules = []
sol_detail = ("Resource CREATE failed: ResourceInError: resources."
"VDU1-0.resources.VDU1: Went to status ERROR due to "
@@ -4245,9 +4288,10 @@ class TestOpenstack(base.BaseTestCase):
mock_update_stack.side_effect = _retry
# execute
self.assertRaises(sol_ex.StackOperationFailed,
self.driver._update_stack_retry, heat_client, fields_example,
inst, STACK_ID, error_ex, vim_info, vdu_ids)
self.assertRaises(
sol_ex.StackOperationFailed, self.driver._update_stack_retry,
heat_client, fields_example, inst, STACK_ID, error_ex, vim_info,
vdu_ids, anti_rules)
self.assertEqual(CONF.v2_vnfm.placement_az_select_retry,
mock_update_stack.call_count)
@@ -4267,76 +4311,10 @@ class TestOpenstack(base.BaseTestCase):
objects.VnfInstanceV2_InstantiatedVnfInfo.from_dict(
_update_retry_instantiated_vnfinfo))
)
fields_example = copy.deepcopy(_fields_example_scale)
heat_client = openstack.heat_utils.HeatClient(vim_info)
vdu_ids = {"VDU1-1", "VDU1-2"}
sol_detail = ("Resource CREATE failed: ResourceInError: resources."
"VDU1-1.resources.VDU1: Went to status ERROR due to "
"\"Message: No valid host was found. , Code: 500\"")
error_ex = sol_ex.StackOperationFailed(sol_detail=sol_detail,
sol_title="stack failed")
mock_get_zone.return_value = {'az-1', 'az-2', 'az-3', 'az-4'}
use_zone_list = []
def _retry(stack_name, fields):
vdu_dict = fields['parameters']['nfv']['VDU']
use_zone = {vdu_id: parameters.get('locationConstraints')
for vdu_id, parameters in vdu_dict.items()
if parameters.get('locationConstraints') is not None}
use_zone_list.append(use_zone)
if mock_update_stack.call_count >= 2:
return
else:
sol_detail = ("Resource UPDATE failed: resources.VDU1-1: "
"Resource CREATE failed: ResourceInError: "
"resources.VDU1: Went to status ERROR due to "
"\"Message: No valid host was found. , "
"Code: 500\"")
raise sol_ex.StackOperationFailed(sol_detail=sol_detail,
sol_title="stack failed")
mock_update_stack.side_effect = _retry
used_zone = 'az-2'
# execute
self.driver._update_stack_retry(heat_client, fields_example, inst,
STACK_ID, error_ex, vim_info, vdu_ids)
self.assertEqual(2, mock_update_stack.call_count)
self.assertEqual(use_zone_list[0]['VDU1-1'],
use_zone_list[0]['VDU1-2'])
self.assertEqual(use_zone_list[1]['VDU1-1'],
use_zone_list[1]['VDU1-2'])
self.assertNotEqual(use_zone_list[0]['VDU1-0'],
use_zone_list[0]['VDU1-1'])
self.assertNotEqual(use_zone_list[1]['VDU1-0'],
use_zone_list[1]['VDU1-1'])
self.assertNotEqual(used_zone, use_zone_list[0]['VDU1-1'])
self.assertNotEqual(used_zone, use_zone_list[1]['VDU1-1'])
@mock.patch.object(openstack.heat_utils.HeatClient, 'update_stack')
@mock.patch.object(openstack.nova_utils.NovaClient, 'get_zone')
def test_update_stack_retry_use_used_zone(self, mock_get_zone,
mock_update_stack):
# prepare
CONF.v2_vnfm.placement_fallback_best_effort = True
vim_info = objects.VimConnectionInfo.from_dict(
_vim_connection_info_example)
inst = objects.VnfInstanceV2(
# required fields
id=uuidutils.generate_uuid(),
instantiatedVnfInfo=(
objects.VnfInstanceV2_InstantiatedVnfInfo.from_dict(
_update_retry_instantiated_vnfinfo))
)
fields_example = copy.deepcopy(_fields_example_scale)
heat_client = openstack.heat_utils.HeatClient(vim_info)
vdu_ids = {"VDU1-1", "VDU1-2"}
anti_rules = [(['VDU1', 'VDU2'], 'zone')]
sol_detail = ("Resource CREATE failed: ResourceInError: resources."
"VDU1-1.resources.VDU1: Went to status ERROR due to "
@@ -4365,15 +4343,61 @@ class TestOpenstack(base.BaseTestCase):
sol_title="stack failed")
mock_update_stack.side_effect = _retry
expected_zone = 'az-2'
# execute
self.driver._update_stack_retry(heat_client, fields_example, inst,
STACK_ID, error_ex, vim_info, vdu_ids)
self.driver._update_stack_retry(
heat_client, fields_example, inst, STACK_ID, error_ex, vim_info,
vdu_ids, anti_rules)
self.assertEqual(3, mock_update_stack.call_count)
self.assertEqual(expected_zone, use_zone_list[2]['VDU1-1'])
self.assertEqual(use_zone_list[0]['VDU1-1'],
use_zone_list[0]['VDU1-2'])
self.assertEqual(use_zone_list[1]['VDU1-1'],
use_zone_list[1]['VDU1-2'])
self.assertEqual(use_zone_list[2]['VDU1-1'],
use_zone_list[2]['VDU1-2'])
self.assertNotEqual(use_zone_list[0]['VDU1-0'],
use_zone_list[0]['VDU1-1'])
self.assertNotEqual(use_zone_list[1]['VDU1-0'],
use_zone_list[1]['VDU1-1'])
self.assertNotEqual(use_zone_list[2]['VDU1-0'],
use_zone_list[2]['VDU1-1'])
# Check for excluded zone used in VDU2-0
expected_zone = 'az-2'
self.assertNotEqual(expected_zone, use_zone_list[0]['VDU1-1'])
self.assertNotEqual(expected_zone, use_zone_list[1]['VDU1-1'])
self.assertEqual(expected_zone, use_zone_list[2]['VDU1-1'])
def test_get_exclude_zone(self):
# prepare
inst = objects.VnfInstanceV2(
id=uuidutils.generate_uuid(),
instantiatedVnfInfo=(
objects.VnfInstanceV2_InstantiatedVnfInfo.from_dict(
_update_retry_instantiated_vnfinfo))
)
failed_vdu_id = 'VDU1-1'
vdu_ids = {"VDU1-1", "VDU1-2"}
vdu_dict = _vdu_dict_example_get_exclude_zone
# Check exclude_zones
# The target members of the Anti-Affinity rule are VDU1 and VDU2.
anti_rules = [(['VDU1', 'VDU2'], 'zone')]
# execute
result = self.driver._get_exclude_zone(
inst, anti_rules, failed_vdu_id, vdu_ids, vdu_dict)
self.assertEqual({'az-2'}, result)
# Check exclude_zones
# The target member of the Anti-Affinity rule is only VDU1.
anti_rules = [(['VDU1'], 'zone')]
# execute
result = self.driver._get_exclude_zone(
inst, anti_rules, failed_vdu_id, vdu_ids, vdu_dict)
self.assertEqual({'az-1', 'az-3', 'az-4'}, result)
@mock.patch.object(openstack.heat_utils.HeatClient, 'get_stack_id')
@mock.patch.object(openstack.heat_utils.HeatClient, 'get_status')