openstack
/
tacker
Code Issues Proposed changes

Fix for multiple failures in VNF lifecycle 

In this patch, multiple failures in VNF lifecycle such as Scale,
Rollback and Heal are fixed.

The fixed cases are the following:
- It cannot set correct value to operation_params due to the lack
  of parameter setting in sending Notification.
- 409 error occurs due to incorrect STATUS setting in Rollback.
- Scale fails since instance_id/aspect_id cannot be set correctly.
- HOT creation in Scale fails since API information cannot be set
  correctly.
- In the case including recreating stack such as Heal, a waiting
  process fails due to un-updated instance_id.
- Scale/Rollback fail since instance_id cannot be held during
  Instantiation.
- Rollback fails due to incorrect state holding during
  Instantiation.
- Uploading package in separate mode fails due to incorrect
  package extraction.

Additionally, the following modifications are also done.
- instance_id setting process is enhanced.
- Mismatches of definition in some arguments are resolved.
- Incorrectly merged code into the master is fixed.
- Setting of STATUS name during Rollback is fixed.
- Mock is added due to the modification of this patch.
- Definition of column in DB migration is fixed.

Change-Id: If538956c433e803149ad109672eed678dcae63e9
Closes-bug: #1902194
This commit is contained in:
Koichi Edagawa 2020-10-30 18:20:18 +09:00
parent da23543bb8
commit bf51fb25a4
13 changed files with 237 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tacker/api/views/vnf_lcm.py
View File

@ -170,11 +170,11 @@ class ViewBuilder(base.BaseViewBuilder):
return {
'id': vnf_lcm_subscription.id,
'filter': filter_dict,
'callbackUri': vnf_lcm_subscription.callback_uri.decode(),
'callbackUri': vnf_lcm_subscription.callback_uri,
}
return {
'id': vnf_lcm_subscription.id,
'callbackUri': vnf_lcm_subscription.callback_uri.decode(),
'callbackUri': vnf_lcm_subscription.callback_uri,
}
else:
return {

38
tacker/api/vnflcm/v1/controller.py
View File

@ -256,18 +256,10 @@ class VnfLcmController(wsgi.Controller):
raise webob.exc.HTTPBadRequest(explanation=msg)
def _notification_process(self, context, vnf_instance,
lcm_operation, request, is_auto=False):
lcm_operation, request, body, is_auto=False):
vnf_lcm_op_occs_id = uuidutils.generate_uuid()
error_point = 0
if lcm_operation == fields.LcmOccsOperationType.HEAL:
request_dict = {
'vnfc_instance_id': request.vnfc_instance_id,
'cause': request.cause
}
operation_params = str(request_dict)
else:
# lcm is instantiation by default
operation_params = str(request.additional_params)
operation_params = jsonutils.dumps(body)
try:
# call create lcm op occs here
LOG.debug('Create LCM OP OCCS')
@ -466,8 +458,7 @@ class VnfLcmController(wsgi.Controller):
except nfvo.VimDefaultNotDefined as exc:
raise webob.exc.HTTPBadRequest(explanation=str(exc))
except(sqlexc.SQLAlchemyError, Exception)\
as exc:
except(sqlexc.SQLAlchemyError, Exception) as exc:
raise webob.exc.HTTPInternalServerError(
explanation=str(exc))
except webob.exc.HTTPNotFound as e:
@ -584,7 +575,7 @@ class VnfLcmController(wsgi.Controller):
vnf_lcm_op_occs_id = \
self._notification_process(context, vnf_instance,
fields.LcmOccsOperationType.INSTANTIATE,
instantiate_vnf_request)
instantiate_vnf_request, request_body)
self.rpc_api.instantiate(context, vnf_instance, vnf,
instantiate_vnf_request, vnf_lcm_op_occs_id)
@ -618,7 +609,7 @@ class VnfLcmController(wsgi.Controller):
vnf_lcm_op_occs_id = \
self._notification_process(context, vnf_instance,
fields.LcmOccsOperationType.TERMINATE,
terminate_vnf_req)
terminate_vnf_req, request_body)
self.rpc_api.terminate(context, vnf_instance, vnf,
terminate_vnf_req, vnf_lcm_op_occs_id)
@ -664,7 +655,7 @@ class VnfLcmController(wsgi.Controller):
vnf_lcm_op_occs_id = \
self._notification_process(context, vnf_instance,
fields.LcmOccsOperationType.HEAL,
heal_vnf_request)
heal_vnf_request, request_body)
self.rpc_api.heal(context, vnf_instance, vnf_dict, heal_vnf_request,
vnf_lcm_op_occs_id)
@ -1061,8 +1052,7 @@ class VnfLcmController(wsgi.Controller):
vnf_info['vnf_lcm_op_occ'] = vnf_lcm_op_occ
vnf_info['after_scale_level'] = scale_level
vnf_info['scale_level'] = current_level
self.rpc_api.scale(context, vnf_info, vnf_instance, scale_vnf_request)
vnf_info['instance_id'] = inst_vnf_info.instance_id
notification = {}
notification['notificationType'] = \
@ -1079,9 +1069,9 @@ class VnfLcmController(wsgi.Controller):
notification['_links']['vnfInstance']['href'] = insta_url
notification['_links']['vnfLcmOpOcc'] = {}
notification['_links']['vnfLcmOpOcc']['href'] = vnflcm_url
self.rpc_api.send_notification(context, notification)
vnf_info['notification'] = notification
self.rpc_api.send_notification(context, notification)
self.rpc_api.scale(context, vnf_info, vnf_instance, scale_vnf_request)
res = webob.Response()
res.status_int = 202
@ -1159,12 +1149,12 @@ class VnfLcmController(wsgi.Controller):
409,
title='OperationState IS NOT FAILED_TEMP')
if vnf_lcm_op_occs.operation != 'INSTANTIATION' \
if vnf_lcm_op_occs.operation != 'INSTANTIATE' \
and vnf_lcm_op_occs.operation != 'SCALE':
return self._make_problem_detail(
'OPERATION IS NOT INSTANTIATION/SCALE',
'OPERATION IS NOT INSTANTIATE/SCALE',
409,
title='OPERATION IS NOT INSTANTIATION/SCALE')
title='OPERATION IS NOT INSTANTIATE/SCALE')
operation_params = jsonutils.loads(
vnf_lcm_op_occs.operation_params)
@ -1180,6 +1170,10 @@ class VnfLcmController(wsgi.Controller):
vnf_instance = self._get_vnf_instance(
context, vnf_lcm_op_occs.vnf_instance_id)
inst_vnf_info = vnf_instance.instantiated_vnf_info
if inst_vnf_info is not None:
vnf_info['instance_id'] = inst_vnf_info.instance_id
vnf_lcm_op_occs.changed_info = None
vnf_info['vnf_lcm_op_occ'] = vnf_lcm_op_occs
return self._rollback(

19
tacker/conductor/conductor_server.py
View File

@ -641,8 +641,8 @@ class Conductor(manager.Manager):
updated_values['status'], vnf_model.status))
vnf_model.update(updated_values)
def _update_vnf_attributes(self, context, vnf_dict, current_statuses,
new_status):
def _update_vnf_attributes(self, context, vnf_instance, vnf_dict,
current_statuses, new_status):
with context.session.begin(subtransactions=True):
try:
modified_attributes = {}
@ -661,6 +661,12 @@ class Conductor(manager.Manager):
message='Cannot change status to {} while \
in {}'.format(updated_values['status'],
vnf_model.status))
if hasattr(vnf_instance.instantiated_vnf_info, 'instance_id'):
instance_id = \
vnf_instance.instantiated_vnf_info.instance_id
if instance_id:
# add instance_id info
updated_values.update({'instance_id': instance_id})
vnf_model.update(updated_values)
for key, val in vnf_dict['attributes'].items():
@ -1025,6 +1031,7 @@ class Conductor(manager.Manager):
vim_info, context)
if scale_vnf_request.type == 'SCALE_IN':
vnf_dict['action'] = 'in'
vnf_dict['policy_name'] = scale_vnf_request.aspect_id
reverse = scale_vnf_request.additional_params.get('is_reverse')
region_name = vim_connection_info.access_info.get('region_name')
scale_id_list, scale_name_list, grp_id, res_num = \
@ -1504,7 +1511,7 @@ class Conductor(manager.Manager):
auth_client = auth.auth_manager.get_auth_client(
notification['subscriptionId'])
response = auth_client.post(
line.callback_uri.decode(),
line.callback_uri,
data=json.dumps(notification))
if response.status_code == 204:
LOG.info(
@ -1517,7 +1524,7 @@ class Conductor(manager.Manager):
callback_uri[%s]" %
(notification['id'],
response.status_code,
line.callback_uri.decode()))
line.callback_uri))
LOG.debug(
"retry_wait %s" %
CONF.vnf_lcm.retry_wait)
@ -1580,7 +1587,7 @@ class Conductor(manager.Manager):
instantiate_vnf_req=instantiate_vnf)
vnf_dict['error_point'] = 7
self._update_vnf_attributes(context, vnf_dict,
self._update_vnf_attributes(context, vnf_instance, vnf_dict,
_PENDING_STATUS, _ACTIVE_STATUS)
self.vnflcm_driver._vnf_instance_update(context, vnf_instance,
instantiation_state=fields.VnfInstanceState.
@ -1604,6 +1611,8 @@ class Conductor(manager.Manager):
self._build_instantiated_vnf_info(context, vnf_instance,
instantiate_vnf)
self.vnflcm_driver._vnf_instance_update(context, vnf_instance,
task_state=None)
# Update vnf_lcm_op_occs table and send notification "FAILED_TEMP"
self._send_lcm_op_occ_notification(

2
tacker/db/migration/alembic_migrations/versions/HEAD
View File

@ -1 +1 @@
329cd1619d41
df26c5871f3c

2
tacker/db/migration/alembic_migrations/versions/d25c7c865ce8_mod_vnflcm_subscription.py
View File

@ -81,5 +81,3 @@ def upgrade(active_plugins=None, options=None):
op.add_column('vnf_lcm_op_occs',
sa.Column('deleted_at', sa.DateTime(), nullable=True))
pass

66
tacker/db/migration/alembic_migrations/versions/df26c5871f3c_change_vnf_filter_column_definition.py Normal file
View File

@ -0,0 +1,66 @@
# Copyright 2020 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# flake8: noqa: E402
"""change_vnf_filter_column_definition
Revision ID: df26c5871f3c
Revises: 329cd1619d41
Create Date: 2020-11-13 18:32:46.703342
"""
# revision identifiers, used by Alembic.
revision = 'df26c5871f3c'
down_revision = '329cd1619d41'
from alembic import op
import sqlalchemy as sa
from tacker.db import migration
def upgrade(active_plugins=None, options=None):
# This migration file is to change syntax of "GENERATED ALWAYS AS"
# from 'filter' to `filter`
# TODO(esto-aln): (1) Need to fix SQL statement such that "mediumblob"
# is used instead of "text". Currently, "text" is used as a workaround.
# (2) Need to fix SQL statement to utilize sqlalchemy. Currently, we
# use raw SQL with op.exec as a workaround since op.alter_column does
# not work correctly.
alter_sql_notification_types = "ALTER TABLE vnf_lcm_filters CHANGE \
notification_types notification_types text GENERATED \
ALWAYS AS (json_unquote(json_extract(`filter`,\
'$.notificationTypes'))) VIRTUAL;"
alter_sql_notification_types_len = "ALTER TABLE vnf_lcm_filters CHANGE \
notification_types_len notification_types_len int(11) GENERATED \
ALWAYS AS (ifnull(json_length(`notification_types`),0)) VIRTUAL;"
alter_sql_operation_types = "ALTER TABLE vnf_lcm_filters CHANGE \
operation_types operation_types text GENERATED ALWAYS AS \
(json_unquote(json_extract(`filter`,'$.operationTypes'))) VIRTUAL;"
alter_sql_operation_types_len = "ALTER TABLE vnf_lcm_filters CHANGE \
operation_types_len operation_types_len int(11) GENERATED ALWAYS \
AS (ifnull(json_length(`operation_types`),0)) VIRTUAL;"
op.execute(alter_sql_notification_types)
op.execute(alter_sql_notification_types_len)
op.execute(alter_sql_operation_types)
op.execute(alter_sql_operation_types_len)

1
tacker/objects/vnf_instance.py
View File

@ -484,6 +484,7 @@ class VnfInstance(base.TackerObject, base.TackerPersistentObject,
'vnf_product_name': self.vnf_product_name,
'vnf_software_version': self.vnf_software_version,
'vnfd_version': self.vnfd_version,
'vnf_pkg_id': self.vnf_pkg_id,
'vnf_metadata': self.vnf_metadata}
if (self.instantiation_state == fields.VnfInstanceState.INSTANTIATED

3
tacker/tests/unit/vnflcm/fakes.py
View File

@ -249,6 +249,7 @@ def _fake_vnf_instance_not_instantiated_response(
'vnfdId': uuidsentinel.vnfd_id,
'vnfdVersion': '1.0',
'vnfSoftwareVersion': '1.0',
'vnfPkgId': uuidsentinel.vnf_pkg_id,
'id': uuidsentinel.vnf_instance_id,
'metadata': {'key': 'value'}
}
@ -862,7 +863,7 @@ def vnflcm_rollback_insta(error_point=7):
start_time=datetime.datetime(2000, 1, 1, 1, 1, 1,
tzinfo=iso8601.UTC),
vnf_instance_id=uuidsentinel.vnf_instance_id,
operation='INSTANTIATION',
operation='INSTANTIATE',
operation_state='FAILED_TEMP',
is_automatic_invocation=False,
operation_params='{}',

11
tacker/tests/unit/vnfm/infra_drivers/openstack/test_openstack_driver.py
View File

@ -191,13 +191,22 @@ class TestOpenStack(base.FixturedTestCase):
grant_info=grant_info_test,
vnf_instance=vnf_instance)
@mock.patch('tacker.vnfm.vim_client.VimClient.get_vim')
@mock.patch('tacker.vnfm.infra_drivers.openstack.openstack'
'.OpenStack._format_base_hot')
@mock.patch('tacker.vnflcm.utils.get_base_nest_hot_dict')
@mock.patch('tacker.common.clients.OpenstackClients')
def test_create_grant(self, mock_OpenstackClients_heat,
mock_get_base_hot_dict,
mock_format_base_hot):
mock_format_base_hot,
mock_get_vim):
mock_get_vim.return_value = {
'vim_id': uuidsentinel.vnfd_id,
'vim_type': 'test',
'vim_auth': {'username': 'test', 'password': 'test'},
'placement_attr': {'region': 'TestRegionOne'},
'tenant': 'test'
}
vnf = utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
flavour='simple')
vnf['placement_attr'] = {'region_name': 'dummy_region'}

77
tacker/tosca/utils.py
View File

@ -1294,3 +1294,80 @@ def update_nested_scaling_resources(nested_resources, mgmt_ports, metadata,
yaml.safe_dump(nested_resources_dict)
return nested_tpl
def get_policies_from_dict(vnfd_dict, policy_type=None):
final_policies = dict()
policies = vnfd_dict.get('topology_template', {}).get('policies', {})
for policy in policies:
for policy_name, policy_dict in policy.items():
if policy_type:
if policy_dict.get('type') == policy_type:
final_policies.update({policy_name: policy_dict})
else:
final_policies.update({policy_name: policy_dict})
return final_policies
@log.log
def get_scale_group(vnf_dict, vnfd_dict, inst_req_info):
scaling_group_dict = dict()
data_dict = dict()
if vnf_dict['attributes'].get('scaling_group_names'):
for policy_name, policy_dict in \
get_policies_from_dict(vnfd_dict,
ETSI_SCALING_ASPECT_DELTA).items():
aspect = policy_dict['properties']['aspect']
vdu = policy_dict['targets']
deltas = policy_dict['properties']['deltas']
for delta_key, delta_dict in deltas.items():
num = delta_dict['number_of_instances']
data_dict.update({
aspect: {
'vdu': vdu,
'num': num
}
})
for aspect_name, aspect_dict in data_dict.items():
aspect_policy = \
get_policies_from_dict(vnfd_dict, ETSI_SCALING_ASPECT)
for policy_name, policy_dict in aspect_policy.items():
aspect = policy_dict['properties']['aspects'][aspect_name]
max_level = aspect.get('max_scale_level')
data_dict[aspect_name].update({'maxLevel': max_level})
delta_policy = \
get_policies_from_dict(vnfd_dict, ETSI_INITIAL_DELTA)
for policy_name, policy_dict in delta_policy.items():
for target in policy_dict['targets']:
if target in aspect_dict['vdu']:
delta = policy_dict['properties']['initial_delta']
number_of_instances = delta['number_of_instances']
data_dict[aspect_name].update(
{'initialNum': number_of_instances})
level_policy = \
get_policies_from_dict(vnfd_dict, ETSI_INST_LEVEL)
for policy_name, policy_dict in level_policy.items():
instantiation_level_id = ""
if hasattr(inst_req_info, 'instantiation_level_id'):
instantiation_level_id = \
inst_req_info.instantiation_level_id
if not instantiation_level_id:
instantiation_level_id = \
policy_dict['properties']['default_level']
levels = policy_dict['properties']['levels']
scale_info = levels[instantiation_level_id]['scale_info']
initial_level = scale_info[aspect_name]['scale_level']
increase = aspect_dict['num'] * initial_level
default = aspect_dict['initialNum'] + increase
data_dict[aspect_name].update({'initialLevel': initial_level,
'default': default})
scaling_group_dict.update({'scaleGroupDict': data_dict})
return scaling_group_dict

13
tacker/vnflcm/vnflcm_driver.py
View File

@ -273,7 +273,7 @@ def revert_to_error_rollback(function):
jsonutils.dump_as_bytes(
resource_dict.get(
'affected_virtual_storages'))
self.rpc_api.sendNotification(context, notification)
self.rpc_api.send_notification(context, notification)
except Exception as e:
LOG.warning("Failed to revert scale info for vnf "
"instance %(id)s. Error: %(error)s",
@ -359,8 +359,9 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
error=encodeutils.exception_to_unicode(exp))
if vnf_instance.instantiated_vnf_info and\
not vnf_instance.instantiated_vnf_info.instance_id:
vnf_instance.instantiated_vnf_info.instance_id != instance_id:
vnf_instance.instantiated_vnf_info.instance_id = instance_id
if vnf_dict['attributes'].get('scaling_group_names'):
vnf_instance.instantiated_vnf_info.scale_status = \
vnf_dict['scale_status']
@ -1330,14 +1331,18 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
def _update_vnf_rollback(self, context, vnf_info,
vnf_instance, vnf_lcm_op_occs):
if vnf_lcm_op_occs.operation == 'SCALE':
status = 'ACTIVE'
else:
status = 'INACTIVE'
self._vnfm_plugin._update_vnf_rollback(context, vnf_info,
'ERROR',
'ACTIVE',
status,
vnf_instance=vnf_instance,
vnf_lcm_op_occ=vnf_lcm_op_occs)
def _update_vnf_rollback_status_err(self, context, vnf_info):
self._vnfm_plugin._update_vnf_rollback_status_err(context, vnf_info)
self._vnfm_plugin.update_vnf_rollback_status_err(context, vnf_info)
def _rollback_mgmt_call(self, context, vnf_info, kwargs):
self._vnfm_plugin.mgmt_call(context, vnf_info, kwargs)

50
tacker/vnfm/infra_drivers/openstack/openstack.py
View File

@ -40,6 +40,7 @@ from tacker.extensions import vnfm
from tacker import objects
from tacker.objects import fields
from tacker.plugins.common import constants
from tacker.tosca import utils as tosca_utils
from tacker.tosca.utils import represent_odict
from tacker.vnflcm import utils as vnflcm_utils
from tacker.vnfm.infra_drivers import abstract_driver
@ -136,9 +137,14 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
LOG.debug('vnf %s', vnf)
if vnf.get('grant'):
if vnf['grant'].vim_connections:
vim_con = vnf['grant'].vim_connections[0]
auth_attr = vim_con.access_info
region_name = auth_attr.get('region')
vim_info = vnflcm_utils._get_vim(
context, vnf['grant'].vim_connections)
vim_connection_info = \
objects.VimConnectionInfo.obj_from_primitive(
vim_info, context)
auth_attr = vim_connection_info.access_info
region_name = \
vim_connection_info.access_info.get('region_name')
else:
region_name = vnf.get('placement_attr', {}).\
get('region_name', None)
@ -183,6 +189,18 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
vnfd_dict = yaml.safe_load(vnfd_str)
LOG.debug('VNFD: %s', vnfd_dict)
LOG.debug('VNF package path: %s', vnf_package_path)
scaling_group_dict = {}
for name, rsc in base_hot_dict.get('resources').items():
if rsc['type'] == 'OS::Heat::AutoScalingGroup':
key_name = name.replace('_group', '')
scaling_group_dict[key_name] = name
if scaling_group_dict:
vnf['attributes']['scaling_group_names'] = \
jsonutils.dump_as_bytes(scaling_group_dict)
scale_group_dict = \
tosca_utils.get_scale_group(vnf, vnfd_dict, inst_req_info)
vnf['attributes']['scale_group'] = \
jsonutils.dump_as_bytes(scale_group_dict)
sys.path.append(vnf_package_path)
user_data_module = os.path.splitext(
user_data_path.lstrip('./'))[0].replace('/', '.')
@ -236,12 +254,18 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
"is not in dict format.")
raise vnfm.LCMUserDataFailed(reason=error_reason)
if vnf['attributes'].get('scale_group'):
scale_json = vnf['attributes']['scale_group']
scaleGroupDict = jsonutils.loads(scale_json)
for name, value in scaleGroupDict['scaleGroupDict'].items():
hot_param_dict[name + '_desired_capacity'] = \
value['default']
if scaling_group_dict:
scale_status_list = []
for name, value in scale_group_dict['scaleGroupDict'].items():
key_name = name + '_desired_capacity'
if base_hot_dict.get('parameters') and \
base_hot_dict['parameters'].get(key_name):
hot_param_dict[key_name] = value['default']
scale_status = objects.ScaleInfo(
aspect_id=name,
scale_level=value['initialLevel'])
scale_status_list.append(scale_status)
vnf['scale_status'] = scale_status_list
if vnf.get('grant'):
grant = vnf['grant']
ins_inf = vnf_instance.instantiated_vnf_info.vnfc_resource_info
@ -635,8 +659,8 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
'status': rsc.resource_status}
LOG.warning(error_reason)
raise vnfm.VNFScaleWaitFailed(
vnf_id=policy['vnf']['\
id'], reason=error_reason)
vnf_id=policy['vnf']['id'],
reason=error_reason)
events = heatclient.resource_event_list(
stack_id, policy_name, limit=1,
sort_dir='desc',
@ -996,7 +1020,9 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
vnfc_res_info.compute_resource, pop_resources,
vnfc_res_info.vdu_id, vim_connection_info)
vnfc_res_info.metadata.update({"stack_id": stack_id})
if stack_id:
vnfc_res_info.metadata.update({"stack_id": stack_id})
_populate_virtual_storage(vnfc_res_info, pop_resources)
# Find out associated VLs, and CP used by vdu_id

2
tacker/vnfm/nfvo_client.py
View File

@ -197,7 +197,7 @@ class VnfPackageRequest:
for pipeline_type in cfg.CONF.connect_vnf_packages.pipeline:
download_vnf_package(pipeline_type, vnf_package_zip)
zip_buffer.seek(0)
zip_buffer.seek(0)
return zip_buffer
@classmethod