Support for Retry Operation

This feature will enable the client to use
this API to initiate retrying a VNF lifecycle
operation.

It is based on the ETSI NFV specification that
retry can be used when a VNF lifecycle operation
is in a FAILED_TEMP state.

Implements: blueprint support-error-handling
Spec: https://specs.openstack.org/openstack/tacker-specs/specs/wallaby/support-error-handling-based-on-ETSI-NFV.html
Change-Id: I96546338fba9946d8df3ffc7ab0003c5b44be85b

api-ref/source/v1/vnflcm.inc

@@ -1119,3 +1119,40 @@ Response Example
 
 .. literalinclude:: samples/vnflcm/list-subscription-response.json
    :language: javascript
+
+Retry
+=================
+
+.. rest_method::  POST /vnflcm/v1/vnf_lcm_op_occs/{vnfLcmOpOccId}/retry
+
+The POST method initiates retrying a VNF lifecycle operation if that
+operation has experienced a temporary failure, i.e. the related
+"Individual VNF LCM operation occurrence" resource
+is in "FAILED_TEMP" state.
+
+In case of success of processing the asynchronous request,
+the "operationState" attribute in the representation of the
+parent resource shall be changed to "PROCESSING" and
+the applicable "start" notification shall be emitted to indicate
+that the underlying VNF LCM operation occurrence proceeds.
+
+Response Codes
+--------------
+
+.. rest_status_code:: success status.yaml
+
+  - 202
+
+.. rest_status_code:: error status.yaml
+
+  - 401
+  - 403
+  - 404
+  - 409
+
+Request Parameters
+------------------
+
+.. rest_parameters:: parameters_vnflcm.yaml
+
+  - vnfLcmOpOccId: vnf_lcm_op_occ_id

releasenotes/notes/errorhandling_fail_retry-31a42c3c7877e0e8.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - Add new RESTful APIs for Fail VNF, Retry VNF as part of error handling
+    operation based on ETSI NFV specifications.

tacker/api/vnflcm/v1/controller.py

@@ -127,6 +127,36 @@ def check_vnf_status(action, status=None):
     return outer
 
 
+def check_vnf_status_and_error_point(action, status=None):
+    """Decorator to check vnf status are valid for particular action.
+
+    If the vnf has the wrong status with the wrong error point,
+    it will raise conflict exception.
+    """
+
+    if status is not None and not isinstance(status, set):
+        status = set(status)
+
+    def outer(f):
+        @functools.wraps(f)
+        def inner(self, context, vnf_instance, vnf, *args, **kw):
+            vnf['current_error_point'] = fields.ErrorPoint.INITIAL
+
+            if 'before_error_point' not in vnf:
+                vnf['before_error_point'] = fields.ErrorPoint.INITIAL
+
+            if status is not None and vnf['status'] not in status and \
+                    vnf['before_error_point'] == fields.ErrorPoint.INITIAL:
+                raise exceptions.VnfConflictStateWithErrorPoint(
+                    uuid=vnf['id'],
+                    state=vnf['status'],
+                    action=action,
+                    error_point=vnf['before_error_point'])
+            return f(self, context, vnf_instance, vnf, *args, **kw)
+        return inner
+    return outer
+
+
 class VnfLcmController(wsgi.Controller):
 
     notification_type_list = ['VnfLcmOperationOccurrenceNotification',
@@ -573,11 +603,13 @@ class VnfLcmController(wsgi.Controller):
     @check_vnf_state(action="instantiate",
         instantiation_state=[fields.VnfInstanceState.NOT_INSTANTIATED],
         task_state=[None])
-    @check_vnf_status(action="instantiate",
+    @check_vnf_status_and_error_point(action="instantiate",
         status=[constants.INACTIVE])
     def _instantiate(self, context, vnf_instance, vnf, request_body):
         req_body = utils.convert_camelcase_to_snakecase(request_body)
 
+        vnf_lcm_op_occs_id = vnf.get('vnf_lcm_op_occs_id')
+
         try:
             self._validate_flavour_and_inst_level(context, req_body,
                                                   vnf_instance)
@@ -595,12 +627,16 @@ class VnfLcmController(wsgi.Controller):
         vnf_instance.save()
 
         # lcm op process
-        vnf_lcm_op_occs_id = \
+        if vnf['before_error_point'] == fields.ErrorPoint.INITIAL:
-            self._notification_process(context, vnf_instance,
+            vnf_lcm_op_occs_id = \
-                                       fields.LcmOccsOperationType.INSTANTIATE,
+                self._notification_process(context, vnf_instance,
-                                       instantiate_vnf_request, request_body)
+                            fields.LcmOccsOperationType.INSTANTIATE,
-        self.rpc_api.instantiate(context, vnf_instance, vnf,
+                            instantiate_vnf_request, request_body)
-                                 instantiate_vnf_request, vnf_lcm_op_occs_id)
+
+        if vnf_lcm_op_occs_id:
+            self.rpc_api.instantiate(context, vnf_instance, vnf,
+                                    instantiate_vnf_request,
+                                    vnf_lcm_op_occs_id)
 
     @wsgi.response(http_client.ACCEPTED)
     @wsgi.expected_errors((http_client.FORBIDDEN, http_client.NOT_FOUND,
@@ -617,9 +653,12 @@ class VnfLcmController(wsgi.Controller):
 
     @check_vnf_state(action="terminate",
                      instantiation_state=[
-                         fields.VnfInstanceState.INSTANTIATED],
+                         fields.VnfInstanceState.INSTANTIATED,
+                         fields.VnfInstanceState.NOT_INSTANTIATED],
                      task_state=[None])
-    def _terminate(self, context, vnf_instance, request_body, vnf):
+    @check_vnf_status_and_error_point(action="terminate",
+        status=[constants.ACTIVE])
+    def _terminate(self, context, vnf_instance, vnf, request_body):
         req_body = utils.convert_camelcase_to_snakecase(request_body)
         terminate_vnf_req = \
             objects.TerminateVnfRequest.obj_from_primitive(
@@ -628,14 +667,18 @@ class VnfLcmController(wsgi.Controller):
         vnf_instance.task_state = fields.VnfInstanceTaskState.TERMINATING
         vnf_instance.save()
 
-        # lcm op process
+        vnf_lcm_op_occs_id = vnf.get('vnf_lcm_op_occs_id')
-        vnf_lcm_op_occs_id = \
-            self._notification_process(context, vnf_instance,
-                                       fields.LcmOccsOperationType.TERMINATE,
-                                       terminate_vnf_req, request_body)
 
-        self.rpc_api.terminate(context, vnf_instance, vnf,
+        # lcm op process
-                               terminate_vnf_req, vnf_lcm_op_occs_id)
+        if vnf['before_error_point'] == fields.ErrorPoint.INITIAL:
+            vnf_lcm_op_occs_id = \
+                self._notification_process(context, vnf_instance,
+                                        fields.LcmOccsOperationType.TERMINATE,
+                                        terminate_vnf_req, request_body)
+
+        if vnf_lcm_op_occs_id:
+            self.rpc_api.terminate(context, vnf_instance, vnf,
+                                terminate_vnf_req, vnf_lcm_op_occs_id)
 
     @wsgi.response(http_client.ACCEPTED)
     @wsgi.expected_errors((http_client.BAD_REQUEST, http_client.FORBIDDEN,
@@ -647,12 +690,9 @@ class VnfLcmController(wsgi.Controller):
 
         vnf = self._get_vnf(context, id)
         vnf_instance = self._get_vnf_instance(context, id)
-        self._terminate(context, vnf_instance, body, vnf)
+        self._terminate(context, vnf_instance, vnf, body)
 
-    @check_vnf_state(action="heal",
+    @check_vnf_status_and_error_point(action="heal",
-        instantiation_state=[fields.VnfInstanceState.INSTANTIATED],
-        task_state=[None])
-    @check_vnf_status(action="heal",
         status=[constants.ACTIVE])
     def _heal(self, context, vnf_instance, vnf_dict, request_body):
         req_body = utils.convert_camelcase_to_snakecase(request_body)
@@ -662,6 +702,8 @@ class VnfLcmController(wsgi.Controller):
             vnfc_resource_info.id for vnfc_resource_info in
             inst_vnf_info.vnfc_resource_info]
 
+        vnf_lcm_op_occs_id = vnf_dict.get('vnf_lcm_op_occs_id')
+
         for vnfc_id in heal_vnf_request.vnfc_instance_id:
             # check if vnfc_id exists in vnfc_resource_info
             if vnfc_id not in vnfc_resource_info_ids:
@@ -675,13 +717,15 @@ class VnfLcmController(wsgi.Controller):
         vnf_instance.save()
 
         # call notification process
-        vnf_lcm_op_occs_id = \
+        if vnf_dict['before_error_point'] == fields.ErrorPoint.INITIAL:
-            self._notification_process(context, vnf_instance,
+            vnf_lcm_op_occs_id = \
-                                       fields.LcmOccsOperationType.HEAL,
+                self._notification_process(context, vnf_instance,
-                                       heal_vnf_request, request_body)
+                                        fields.LcmOccsOperationType.HEAL,
+                                        heal_vnf_request, request_body)
 
-        self.rpc_api.heal(context, vnf_instance, vnf_dict, heal_vnf_request,
+        if vnf_lcm_op_occs_id:
-                          vnf_lcm_op_occs_id)
+            self.rpc_api.heal(context, vnf_instance, vnf_dict,
+                            heal_vnf_request, vnf_lcm_op_occs_id)
 
     @wsgi.response(http_client.ACCEPTED)
     @wsgi.expected_errors((http_client.BAD_REQUEST, http_client.FORBIDDEN,
@@ -693,6 +737,21 @@ class VnfLcmController(wsgi.Controller):
 
         vnf = self._get_vnf(context, id)
         vnf_instance = self._get_vnf_instance(context, id)
+
+        if vnf_instance.instantiation_state not in \
+           [fields.VnfInstanceState.INSTANTIATED]:
+            raise exceptions.VnfInstanceConflictState(
+                attr='instantiation_state',
+                uuid=vnf_instance.id,
+                state=vnf_instance.instantiation_state,
+                action='heal')
+        if vnf_instance.task_state not in [None]:
+            raise exceptions.VnfInstanceConflictState(
+                attr='task_state',
+                uuid=vnf_instance.id,
+                state=vnf_instance.task_state,
+                action='heal')
+
         self._heal(context, vnf_instance, vnf, body)
 
     @wsgi.response(http_client.OK)
@@ -1017,12 +1076,17 @@ class VnfLcmController(wsgi.Controller):
     @check_vnf_state(action="scale",
         instantiation_state=[fields.VnfInstanceState.INSTANTIATED],
         task_state=[None])
+    @check_vnf_status_and_error_point(action="scale",
+        status=[constants.ACTIVE])
     def _scale(self, context, vnf_instance, vnf_info, request_body):
         req_body = utils.convert_camelcase_to_snakecase(request_body)
         scale_vnf_request = objects.ScaleVnfRequest.obj_from_primitive(
             req_body, context=context)
         inst_vnf_info = vnf_instance.instantiated_vnf_info
 
+        if 'vnf_lcm_op_occs_id' in vnf_info:
+            vnf_lcm_op_occs_id = vnf_info['vnf_lcm_op_occs_id']
+
         aspect = False
         current_level = 0
         for scale in inst_vnf_info.scale_status:
@@ -1077,27 +1141,47 @@ class VnfLcmController(wsgi.Controller):
             if max_level < scale_level:
                 return self._make_problem_detail(
                     'can not scale_out', 400, title='can not scale_out')
+            if 'vnf_lcm_op_occs_id' in vnf_info:
+                num = (scaleGroupDict['scaleGroupDict']
+                    [scale_vnf_request.aspect_id]['num'])
+                default = (scaleGroupDict['scaleGroupDict']
+                    [scale_vnf_request.aspect_id]['default'])
+                vnf_info['res_num'] = (num *
+                    scale_vnf_request.number_of_steps + default)
 
-        vnf_lcm_op_occs_id = uuidutils.generate_uuid()
+        if vnf_info['before_error_point'] == fields.ErrorPoint.INITIAL:
-        timestamp = datetime.datetime.utcnow()
+            vnf_lcm_op_occs_id = uuidutils.generate_uuid()
-        operation_params = {
+            timestamp = datetime.datetime.utcnow()
-            'type': scale_vnf_request.type,
+            operation_params = {
-            'aspect_id': scale_vnf_request.aspect_id,
+                'type': scale_vnf_request.type,
-            'number_of_steps': scale_vnf_request.number_of_steps,
+                'aspect_id': scale_vnf_request.aspect_id,
-            'additional_params': scale_vnf_request.additional_params}
+                'number_of_steps': scale_vnf_request.number_of_steps,
-        vnf_lcm_op_occ = objects.VnfLcmOpOcc(
+                'additional_params': scale_vnf_request.additional_params}
-            context=context,
+
-            id=vnf_lcm_op_occs_id,
+            vnf_lcm_op_occ = objects.VnfLcmOpOcc(
-            operation_state='STARTING',
+                context=context,
-            state_entered_time=timestamp,
+                id=vnf_lcm_op_occs_id,
-            start_time=timestamp,
+                operation_state='STARTING',
-            vnf_instance_id=inst_vnf_info.vnf_instance_id,
+                state_entered_time=timestamp,
-            operation='SCALE',
+                start_time=timestamp,
-            is_automatic_invocation=scale_vnf_request.additional_params.get('\
+                vnf_instance_id=inst_vnf_info.vnf_instance_id,
-                is_auto'),
+                operation='SCALE',
-            operation_params=json.dumps(operation_params),
+                is_automatic_invocation=scale_vnf_request.additional_params.get('\
-            error_point=1)
+                    is_auto'),
-        vnf_lcm_op_occ.create()
+                operation_params=json.dumps(operation_params),
+                error_point=1)
+            vnf_lcm_op_occ.create()
+        else:
+            try:
+                vnf_lcm_op_occ = objects.VnfLcmOpOcc.get_by_id(
+                    context, vnf_lcm_op_occs_id)
+            except exceptions.NotFound as lcm_e:
+                return self._make_problem_detail(str(lcm_e),
+                    404, title='Not Found')
+            except (sqlexc.SQLAlchemyError, Exception) as exc:
+                LOG.exception(exc)
+                return self._make_problem_detail(str(exc),
+                    500, title='Internal Server Error')
 
         vnf_instance.task_state = fields.VnfInstanceTaskState.SCALING
         vnf_instance.save()
@@ -1129,7 +1213,9 @@ class VnfLcmController(wsgi.Controller):
         notification['_links']['vnfLcmOpOcc'] = {}
         notification['_links']['vnfLcmOpOcc']['href'] = vnflcm_url
         vnf_info['notification'] = notification
-        self.rpc_api.send_notification(context, notification)
+
+        if vnf_info['before_error_point'] == fields.ErrorPoint.INITIAL:
+            self.rpc_api.send_notification(context, notification)
         self.rpc_api.scale(context, vnf_info, vnf_instance, scale_vnf_request)
 
         res = webob.Response()
@@ -1350,6 +1436,73 @@ class VnfLcmController(wsgi.Controller):
 
         return self._view_builder.show_lcm_op_occs(vnf_lcm_op_occs)
 
+    @wsgi.response(http_client.ACCEPTED)
+    @wsgi.expected_errors((http_client.BAD_REQUEST, http_client.FORBIDDEN,
+                           http_client.NOT_FOUND, http_client.CONFLICT))
+    def retry(self, request, id):
+        context = request.environ['tacker.context']
+        context.can(vnf_lcm_policies.VNFLCM % 'retry')
+
+        try:
+            vnf_lcm_op_occs = objects.VnfLcmOpOcc.get_by_id(context, id)
+        except exceptions.NotFound as lcm_e:
+            return self._make_problem_detail(str(lcm_e),
+                404, title='Not Found')
+        except (sqlexc.SQLAlchemyError, Exception) as exc:
+            LOG.exception(exc)
+            return self._make_problem_detail(str(exc),
+                500, title='Internal Server Error')
+
+        # operation state checking
+        if vnf_lcm_op_occs.operation_state != \
+           fields.LcmOccsOperationState.FAILED_TEMP:
+            error_msg = ('Cannot proceed with operation_state %s'
+                % vnf_lcm_op_occs.operation_state)
+            return self._make_problem_detail(error_msg,
+                409, title='Conflict')
+
+        # get vnf
+        try:
+            vnf = self._get_vnf(context, vnf_lcm_op_occs.vnf_instance_id)
+        except webob.exc.HTTPNotFound as lcm_e:
+            return self._make_problem_detail(str(lcm_e),
+                404, title='Not Found')
+        except Exception as exc:
+            LOG.exception(exc)
+            return self._make_problem_detail(str(exc),
+                500, title='Internal Server Error')
+
+        # get vnf instance
+        try:
+            vnf_instance = objects.VnfInstance.get_by_id(
+                context, vnf_lcm_op_occs.vnf_instance_id)
+        except exceptions.VnfInstanceNotFound:
+            msg = (_("Can not find requested vnf instance: %s")
+                % vnf_lcm_op_occs.vnf_instance_id)
+            return self._make_problem_detail(msg,
+                404, title='Not Found')
+        except Exception as exc:
+            LOG.exception(exc)
+            return self._make_problem_detail(str(exc),
+                500, title='Internal Server Error')
+
+        operation = vnf_lcm_op_occs.operation
+        body = jsonutils.loads(vnf_lcm_op_occs.operation_params)
+        vnf['before_error_point'] = vnf_lcm_op_occs.error_point
+        vnf['vnf_lcm_op_occs_id'] = id
+        if operation == fields.LcmOccsOperationType.INSTANTIATE:
+            self._instantiate(context, vnf_instance, vnf, body)
+        elif operation == fields.LcmOccsOperationType.TERMINATE:
+            self._terminate(context, vnf_instance, vnf, body)
+        elif operation == fields.LcmOccsOperationType.HEAL:
+            self._heal(context, vnf_instance, vnf, body)
+        elif operation == fields.LcmOccsOperationType.SCALE:
+            self._scale(context, vnf_instance, vnf, body)
+        else:
+            error_msg = 'Operation type %s is inavalid' % operation
+            return self._make_problem_detail(error_msg,
+                500, title='Internal Server Error')
+
     def _make_problem_detail(
             self,
             detail,

tacker/api/vnflcm/v1/router.py

@@ -125,6 +125,12 @@ class VnflcmAPIRouter(wsgi.Router):
                 "/vnf_lcm_op_occs/{id}/fail",
                 methods, controller, default_resource)
 
+        # {apiRoot}/vnflcm/v1/vnf_lcm_op_occs/{vnfLcmOpOccId}/retry resource
+        methods = {"POST": "retry"}
+        self._setup_route(mapper,
+                "/vnf_lcm_op_occs/{id}/retry",
+                methods, controller, default_resource)
+
         methods = {"GET": "subscription_list", "POST": "register_subscription"}
         self._setup_route(mapper, "/subscriptions",
                 methods, controller, default_resource)

tacker/common/exceptions.py

@@ -406,3 +406,10 @@ class MgmtDriverRemoteCommandTimeOut(TackerException):
 
 class MgmtDriverOtherError(TackerException):
     message = _('An error occurred in MgmtDriver: %(error_message)s.')
+
+
+class VnfConflictStateWithErrorPoint(Conflict):
+    message = _("Vnf %(uuid)s in status %(state)s. "
+                "Error point %(error_point)s. "
+                "Cannot %(action)s while the vnf is in this state "
+                "with this error point.")

tacker/conductor/conductor_server.py

@@ -103,12 +103,15 @@ cfg.CONF.register_opts(OPTS, 'keystone_authtoken')
 
 LOG = logging.getLogger(__name__)
 
-_INACTIVE_STATUS = ('INACTIVE')
+_INACTIVE_STATUS = ('INACTIVE',)
-_ACTIVE_STATUS = ('ACTIVE')
+_ACTIVE_STATUS = ('ACTIVE',)
 _PENDING_STATUS = ('PENDING_CREATE',
                    'PENDING_TERMINATE',
                    'PENDING_DELETE',
                    'PENDING_HEAL')
+_ERROR_STATUS = ('ERROR',)
+_ALL_STATUSES = _ACTIVE_STATUS + _INACTIVE_STATUS + _PENDING_STATUS + \
+    _ERROR_STATUS
 
 
 def _delete_csar(context, vnf_package):
@@ -1559,13 +1562,12 @@ class Conductor(manager.Manager):
             instantiate_vnf,
             vnf_lcm_op_occs_id):
 
-        vnf_dict['error_point'] = 1
+        if vnf_dict['before_error_point'] == fields.ErrorPoint.INITIAL:
-
+            self._instantiate_grant(context,
-        self._instantiate_grant(context,
+                vnf_instance,
-            vnf_instance,
+                vnf_dict,
-            vnf_dict,
+                instantiate_vnf,
-            instantiate_vnf,
+                vnf_lcm_op_occs_id)
-            vnf_lcm_op_occs_id)
 
         try:
             # Update vnf_lcm_op_occs table and send notification "PROCESSING"
@@ -1576,24 +1578,32 @@ class Conductor(manager.Manager):
                 vnf_instance=vnf_instance,
                 request_obj=instantiate_vnf
             )
+            vnf_dict['current_error_point'] = \
+                fields.ErrorPoint.NOTIFY_PROCESSING
 
-            # change vnf_status
+            if vnf_dict['before_error_point'] <= \
-            if vnf_dict['status'] == 'INACTIVE':
+               fields.ErrorPoint.NOTIFY_PROCESSING:
-                vnf_dict['status'] = 'PENDING_CREATE'
+                # change vnf_status
-            self._change_vnf_status(context, vnf_instance.id,
+                if vnf_dict['status'] == 'INACTIVE':
-                            _INACTIVE_STATUS, 'PENDING_CREATE')
+                    vnf_dict['status'] = 'PENDING_CREATE'
+                self._change_vnf_status(context, vnf_instance.id,
+                                _INACTIVE_STATUS, 'PENDING_CREATE')
 
-            vnf_dict['error_point'] = 3
+            if vnf_dict['before_error_point'] <= \
-            self.vnflcm_driver.instantiate_vnf(context, vnf_instance,
+               fields.ErrorPoint.VNF_CONFIG_END:
-                                               vnf_dict, instantiate_vnf)
+                self.vnflcm_driver.instantiate_vnf(context, vnf_instance,
-            vnf_dict['error_point'] = 5
+                                                vnf_dict, instantiate_vnf)
-            self._build_instantiated_vnf_info(context,
+
-                        vnf_instance,
+                self._build_instantiated_vnf_info(context,
-                        instantiate_vnf_req=instantiate_vnf)
+                            vnf_instance,
+                            instantiate_vnf_req=instantiate_vnf)
+
+                self._update_vnf_attributes(context, vnf_instance, vnf_dict,
+                                            _PENDING_STATUS, _ACTIVE_STATUS)
+
+            vnf_dict['current_error_point'] = \
+                fields.ErrorPoint.NOTIFY_COMPLETED
 
-            vnf_dict['error_point'] = 7
-            self._update_vnf_attributes(context, vnf_instance, vnf_dict,
-                                        _PENDING_STATUS, _ACTIVE_STATUS)
             self.vnflcm_driver._vnf_instance_update(context, vnf_instance,
                         instantiation_state=fields.VnfInstanceState.
                         INSTANTIATED, task_state=None)
@@ -1611,8 +1621,14 @@ class Conductor(manager.Manager):
             )
 
         except Exception as ex:
+            if vnf_dict['current_error_point'] == \
+                    fields.ErrorPoint.PRE_VIM_CONTROL:
+                if hasattr(vnf_instance.instantiated_vnf_info, 'instance_id'):
+                    if vnf_instance.instantiated_vnf_info.instance_id:
+                        vnf_dict['current_error_point'] = \
+                            fields.ErrorPoint.POST_VIM_CONTROL
             self._change_vnf_status(context, vnf_instance.id,
-                            _PENDING_STATUS, 'ERROR')
+                            _ALL_STATUSES, 'ERROR')
 
             self._build_instantiated_vnf_info(context, vnf_instance,
                 instantiate_vnf)
@@ -1628,16 +1644,17 @@ class Conductor(manager.Manager):
                 request_obj=instantiate_vnf,
                 operation_state=fields.LcmOccsOperationState.FAILED_TEMP,
                 error=str(ex),
-                error_point=vnf_dict['error_point']
+                error_point=vnf_dict['current_error_point']
             )
 
     @coordination.synchronized('{vnf_instance[id]}')
     def terminate(self, context, vnf_lcm_op_occs_id,
                   vnf_instance, terminate_vnf_req, vnf_dict):
 
-        self._terminate_grant(context,
+        if vnf_dict['before_error_point'] == fields.ErrorPoint.INITIAL:
-            vnf_instance,
+            self._terminate_grant(context,
-            vnf_lcm_op_occs_id)
+                vnf_instance,
+                vnf_lcm_op_occs_id)
 
         try:
             old_vnf_instance = copy.deepcopy(vnf_instance)
@@ -1652,16 +1669,26 @@ class Conductor(manager.Manager):
                 operation=fields.LcmOccsOperationType.TERMINATE
             )
 
-            self._change_vnf_status(context, vnf_instance.id,
+            vnf_dict['current_error_point'] = \
-                            _ACTIVE_STATUS, 'PENDING_TERMINATE')
+                fields.ErrorPoint.NOTIFY_PROCESSING
 
-            self.vnflcm_driver.terminate_vnf(context, vnf_instance,
+            if vnf_dict['before_error_point'] <= \
-                                             terminate_vnf_req)
+               fields.ErrorPoint.NOTIFY_PROCESSING:
+                self._change_vnf_status(context, vnf_instance.id,
+                                _ACTIVE_STATUS, 'PENDING_TERMINATE')
 
-            self._delete_placement(context, vnf_instance.id)
+            if vnf_dict['before_error_point'] <= \
+               fields.ErrorPoint.VNF_CONFIG_END:
+                self.vnflcm_driver.terminate_vnf(context, vnf_instance,
+                                                 terminate_vnf_req, vnf_dict)
 
-            self._change_vnf_status(context, vnf_instance.id,
+                self._delete_placement(context, vnf_instance.id)
-                            _PENDING_STATUS, 'INACTIVE')
+
+                self._change_vnf_status(context, vnf_instance.id,
+                                _PENDING_STATUS, 'INACTIVE')
+
+            vnf_dict['current_error_point'] = \
+                fields.ErrorPoint.NOTIFY_COMPLETED
 
             self.vnflcm_driver._vnf_instance_update(context, vnf_instance,
                 vim_connection_info=[], task_state=None,
@@ -1682,7 +1709,7 @@ class Conductor(manager.Manager):
         except Exception as exc:
             # set vnf_status to error
             self._change_vnf_status(context, vnf_instance.id,
-                            _PENDING_STATUS, 'ERROR')
+                            _ALL_STATUSES, 'ERROR')
 
             # Update vnf_lcm_op_occs table and send notification "FAILED_TEMP"
             self._send_lcm_op_occ_notification(
@@ -1693,7 +1720,8 @@ class Conductor(manager.Manager):
                 request_obj=terminate_vnf_req,
                 operation=fields.LcmOccsOperationType.TERMINATE,
                 operation_state=fields.LcmOccsOperationState.FAILED_TEMP,
-                error=str(exc)
+                error=str(exc),
+                error_point=vnf_dict['current_error_point']
             )
 
     @coordination.synchronized('{vnf_instance[id]}')
@@ -1704,11 +1732,12 @@ class Conductor(manager.Manager):
              heal_vnf_request,
              vnf_lcm_op_occs_id):
 
-        self._heal_grant(context,
+        if vnf_dict['before_error_point'] == fields.ErrorPoint.INITIAL:
-            vnf_instance,
+            self._heal_grant(context,
-            vnf_dict,
+                vnf_instance,
-            heal_vnf_request,
+                vnf_dict,
-            vnf_lcm_op_occs_id)
+                heal_vnf_request,
+                vnf_lcm_op_occs_id)
 
         try:
             old_vnf_instance = copy.deepcopy(vnf_instance)
@@ -1723,19 +1752,32 @@ class Conductor(manager.Manager):
                 operation=fields.LcmOccsOperationType.HEAL
             )
 
-            # update vnf status to PENDING_HEAL
+            vnf_dict['current_error_point'] = \
-            self._change_vnf_status(context, vnf_instance.id,
+                fields.ErrorPoint.NOTIFY_PROCESSING
-                            _ACTIVE_STATUS, constants.PENDING_HEAL)
-            self.vnflcm_driver.heal_vnf(context, vnf_instance,
-                                        vnf_dict, heal_vnf_request)
-            self._update_instantiated_vnf_info(context, vnf_instance,
-                                               heal_vnf_request)
 
-            # update instance_in in vnf_table
+            if vnf_dict['before_error_point'] <= \
-            self._add_additional_vnf_info(context, vnf_instance)
+               fields.ErrorPoint.NOTIFY_PROCESSING:
-            # update vnf status to ACTIVE
+                # update vnf status to PENDING_HEAL
-            self._change_vnf_status(context, vnf_instance.id,
+                self._change_vnf_status(context, vnf_instance.id,
-                            _PENDING_STATUS, constants.ACTIVE)
+                                _ACTIVE_STATUS, constants.PENDING_HEAL)
+
+            if vnf_dict['before_error_point'] <= \
+               fields.ErrorPoint.VNF_CONFIG_END:
+                self.vnflcm_driver.heal_vnf(context, vnf_instance,
+                                            vnf_dict, heal_vnf_request)
+
+                self._update_instantiated_vnf_info(context, vnf_instance,
+                                                   heal_vnf_request)
+
+                # update instance_in in vnf_table
+                self._add_additional_vnf_info(context, vnf_instance)
+
+                # update vnf status to ACTIVE
+                self._change_vnf_status(context, vnf_instance.id,
+                                _PENDING_STATUS, constants.ACTIVE)
+
+            vnf_dict['current_error_point'] = \
+                fields.ErrorPoint.NOTIFY_COMPLETED
 
             # during .save() ,instantiated_vnf_info is also saved to DB
             self.vnflcm_driver._vnf_instance_update(context, vnf_instance,
@@ -1755,8 +1797,8 @@ class Conductor(manager.Manager):
             )
         except Exception as ex:
             # update vnf_status to 'ERROR' and create event with 'ERROR' status
-            self._change_vnf_status(context, vnf_instance,
+            self._change_vnf_status(context, vnf_instance.id,
-                        _PENDING_STATUS, constants.ERROR, str(ex))
+                        _ALL_STATUSES, constants.ERROR, str(ex))
 
             # call _update_instantiated_vnf_info for notification
             self._update_instantiated_vnf_info(context, vnf_instance,
@@ -1771,7 +1813,8 @@ class Conductor(manager.Manager):
                 request_obj=heal_vnf_request,
                 operation=fields.LcmOccsOperationType.HEAL,
                 operation_state=fields.LcmOccsOperationState.FAILED_TEMP,
-                error=str(ex)
+                error=str(ex),
+                error_point=vnf_dict['current_error_point']
             )
 
     @coordination.synchronized('{vnf_instance[id]}')
@@ -1779,12 +1822,13 @@ class Conductor(manager.Manager):
         vnf_lcm_op_occ = vnf_info['vnf_lcm_op_occ']
         vnf_lcm_op_occ_id = vnf_lcm_op_occ.id
 
-        self._scale_grant(
+        if vnf_info['before_error_point'] == fields.ErrorPoint.INITIAL:
-            context,
+            self._scale_grant(
-            vnf_info,
+                context,
-            vnf_instance,
+                vnf_info,
-            scale_vnf_request,
+                vnf_instance,
-            vnf_lcm_op_occ_id)
+                scale_vnf_request,
+                vnf_lcm_op_occ_id)
 
         self.vnflcm_driver.scale_vnf(
             context, vnf_info, vnf_instance, scale_vnf_request)

tacker/objects/fields.py

@@ -233,8 +233,9 @@ class LcmOccsOperationType(BaseTackerEnum):
     INSTANTIATE = 'INSTANTIATE'
     TERMINATE = 'TERMINATE'
     HEAL = 'HEAL'
+    SCALE = 'SCALE'
 
-    ALL = (INSTANTIATE, TERMINATE, HEAL)
+    ALL = (INSTANTIATE, TERMINATE, HEAL, SCALE)
 
 
 class LcmOccsNotificationStatus(BaseTackerEnum):
@@ -269,3 +270,14 @@ class VnfStatus(BaseTackerEnum):
 
 class InstanceOperation(BaseTackerEnum):
     MODIFY_INFO = 'MODIFY_INFO'
+
+
+class ErrorPoint(BaseTackerEnum):
+    INITIAL = 0
+    NOTIFY_PROCESSING = 1
+    VNF_CONFIG_START = 2
+    PRE_VIM_CONTROL = 3
+    POST_VIM_CONTROL = 4
+    INTERNAL_PROCESSING = 5
+    VNF_CONFIG_END = 6
+    NOTIFY_COMPLETED = 7

tacker/objects/vnf_lcm_op_occs.py

@@ -76,6 +76,21 @@ def _vnf_lcm_op_occs_get_by_id(context, vnf_lcm_op_occ_id):
     return result
 
 
+@db_api.context_manager.reader
+def _vnf_lcm_op_occs_get_by_vnf_instance_id(context, vnf_instance_id):
+
+    query = api.model_query(context, models.VnfLcmOpOccs,
+                            read_deleted="no", project_only=True). \
+        filter_by(vnf_instance_id=vnf_instance_id)
+
+    result = query.first()
+
+    if not result:
+        raise exceptions.VnfInstanceNotFound(id=vnf_instance_id)
+
+    return result
+
+
 @db_api.context_manager.reader
 def _vnf_notify_get_by_id(context, vnf_instance_id, columns_to_join=None):
 
@@ -287,6 +302,12 @@ class VnfLcmOpOcc(base.TackerObject, base.TackerObjectDictCompat,
         db_vnf_lcm_op_occs = _vnf_lcm_op_occs_get_by_id(context, id)
         return cls._from_db_object(context, cls(), db_vnf_lcm_op_occs)
 
+    @base.remotable_classmethod
+    def get_by_vnf_instance_id(cls, context, id):
+        db_vnf_lcm_op_occs = _vnf_lcm_op_occs_get_by_vnf_instance_id(
+            context, id)
+        return cls._from_db_object(context, cls(), db_vnf_lcm_op_occs)
+
 
 @base.TackerObjectRegistry.register
 class ResourceChanges(base.TackerObject,
@@ -618,6 +639,9 @@ class VnfInfoModifications(base.TackerObject,
                 VnfInfoModifications, cls).obj_from_primitive(
                 primitive, context)
         else:
+            if isinstance(primitive, str):
+                primitive = jsonutils.loads(primitive)
+
             if 'vim_connection_info' in primitive.keys():
                 obj_data = [objects.VimConnectionInfo._from_dict(
                     vim_conn) for vim_conn in primitive.get(

tacker/policies/vnf_lcm.py

@@ -154,6 +154,17 @@ rules = [
             }
         ]
     ),
+    policy.DocumentedRuleDefault(
+        name=VNFLCM % 'retry',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description="Retry a VNF instance.",
+        operations=[
+            {
+                'method': 'POST',
+                'path': '/vnflcm/v1/vnf_lcm_op_occs/{vnfLcmOpOccId}/retry'
+            }
+        ]
+    ),
 ]
 
 

tacker/tests/functional/sol/vnflcm/base.py

@@ -469,6 +469,15 @@ class BaseVnfLcmTest(base.BaseTackerTest):
 
         return resp, response_body
 
+    def _retry_op_occs(self, vnf_lcm_op_occs_id):
+        retry_url = os.path.join(
+            self.base_vnf_lcm_op_occs_url,
+            vnf_lcm_op_occs_id, 'retry')
+        resp, response_body = self.http_client.do_request(
+            retry_url, "POST")
+
+        return resp, response_body
+
     def _show_op_occs(self, vnf_lcm_op_occs_id):
         show_url = os.path.join(
             self.base_vnf_lcm_op_occs_url,
@@ -1023,6 +1032,27 @@ class BaseVnfLcmTest(base.BaseTackerTest):
             'VnfLcmOperationOccurrenceNotification',
             'FAILED')
 
+    def assert_retry_vnf(self, resp, vnf_instance_id):
+        self.assertEqual(202, resp.status_code)
+
+        # FT-checkpoint: Notification
+        callback_url = os.path.join(
+            MOCK_NOTIFY_CALLBACK_URL,
+            self._testMethodName)
+        notify_mock_responses = self._filter_notify_history(callback_url,
+            vnf_instance_id)
+
+        self.assertEqual(2, len(notify_mock_responses))
+        self.assert_notification_mock_response(
+            notify_mock_responses[0],
+            'VnfLcmOperationOccurrenceNotification',
+            'PROCESSING')
+
+        self.assert_notification_mock_response(
+            notify_mock_responses[1],
+            'VnfLcmOperationOccurrenceNotification',
+            'FAILED_TEMP')
+
     def assert_update_vnf(
             self,
             resp,

tacker/tests/functional/sol/vnflcm/test_vnf_instance_with_user_data.py

@@ -766,6 +766,228 @@ class VnfLcmWithUserDataTest(vnflcm_base.BaseVnfLcmTest):
 
         self._instantiate_vnf_instance_fail(vnf_instance['id'], request_body)
 
+    def test_retry_instantiate(self):
+        """Test retry operation for instantiation.
+
+        In this test case, we do following steps.
+            - Create subscription.
+            - Create VNF package.
+            - Upload VNF package.
+            - Create VNF instance.
+            - Instantiate VNF(Will fail).
+            - Get vnflcmOpOccId to retry.
+            - Retry instantiation operation.
+            - Get opOccs information.
+            - Delete subscription.
+        """
+        # Create subscription and register it.
+        request_body = fake_vnflcm.Subscription.make_create_request_body(
+            'http://localhost:{}{}'.format(
+                vnflcm_base.FAKE_SERVER_MANAGER.SERVER_PORT,
+                os.path.join(vnflcm_base.MOCK_NOTIFY_CALLBACK_URL,
+                    self._testMethodName)))
+        resp, response_body = self._register_subscription(request_body)
+        self.assertEqual(201, resp.status_code)
+        self.assert_http_header_location_for_subscription(resp.headers)
+        subscription_id = response_body.get('id')
+        self.addCleanup(self._delete_subscription, subscription_id)
+
+        # Pre Setting: Create vnf package.
+        sample_name = 'functional3'
+        csar_package_path = os.path.abspath(
+            os.path.join(
+                os.path.dirname(__file__),
+                "../../../etc/samples/etsi/nfv",
+                sample_name))
+        tempname, _ = vnflcm_base._create_csar_with_unique_vnfd_id(
+            csar_package_path)
+        # upload vnf package
+        vnf_package_id, vnfd_id = vnflcm_base._create_and_upload_vnf_package(
+            self.tacker_client, user_defined_data={
+                "key": sample_name}, temp_csar_path=tempname)
+
+        # Post Setting: Reserve deleting vnf package.
+        self.addCleanup(
+            vnflcm_base._delete_vnf_package,
+            self.tacker_client,
+            vnf_package_id)
+
+        # Create vnf instance
+        resp, vnf_instance = self._create_vnf_instance_from_body(
+            fake_vnflcm.VnfInstances.make_create_request_body(vnfd_id))
+        vnf_instance_id = vnf_instance['id']
+        self._wait_lcm_done(vnf_instance_id=vnf_instance_id)
+        self.assert_create_vnf(resp, vnf_instance, vnf_package_id)
+        self.addCleanup(self._delete_vnf_instance, vnf_instance_id)
+
+        # Failed instantiate VNF
+        request_body = fake_vnflcm.VnfInstances.make_inst_request_body(
+            self.vim['tenant_id'], self.ext_networks, self.ext_mngd_networks,
+            self.ext_link_ports, self.ext_subnets)
+        resp, _ = self._instantiate_vnf_instance(vnf_instance_id, request_body)
+        self._wait_lcm_done('FAILED_TEMP', vnf_instance_id=vnf_instance_id)
+
+        callback_url = os.path.join(
+            vnflcm_base.MOCK_NOTIFY_CALLBACK_URL,
+            self._testMethodName)
+        notify_mock_responses = vnflcm_base.FAKE_SERVER_MANAGER.get_history(
+            callback_url)
+        vnflcm_base.FAKE_SERVER_MANAGER.clear_history(
+            callback_url)
+
+        # get vnflcm_op_occ_id
+        vnflcm_op_occ_id = notify_mock_responses[0].request_body.get(
+            'vnfLcmOpOccId')
+        self.assertIsNotNone(vnflcm_op_occ_id)
+
+        # retry
+        resp, _ = self._retry_op_occs(vnflcm_op_occ_id)
+        self._wait_lcm_done('FAILED_TEMP', vnf_instance_id=vnf_instance_id)
+        self.assert_retry_vnf(resp, vnf_instance_id)
+
+        # rollback (Execute because it's needed to delete VNF)
+        resp, _ = self._rollback_op_occs(vnflcm_op_occ_id)
+        self._wait_lcm_done('ROLLING_BACK', vnf_instance_id=vnf_instance_id)
+        self._wait_lcm_done('ROLLED_BACK', vnf_instance_id=vnf_instance_id)
+        self.assert_rollback_vnf(resp, vnf_instance_id)
+
+        # occ-show
+        resp, op_occs_info = self._show_op_occs(vnflcm_op_occ_id)
+        self._assert_occ_show(resp, op_occs_info)
+
+        # Delete VNF
+        resp, _ = self._delete_vnf_instance(vnf_instance_id)
+        self._wait_lcm_done(vnf_instance_id=vnf_instance_id)
+        self.assert_delete_vnf(resp, vnf_instance_id, vnf_package_id)
+
+        # Subscription delete
+        resp, response_body = self._delete_subscription(subscription_id)
+        self.assertEqual(204, resp.status_code)
+
+    def test_retry_scale_out(self):
+        """Test retry operation for Scale-Out operation.
+
+        In this test case, we do following steps.
+            - Create subscription.
+            - Create VNF package.
+            - Upload VNF package.
+            - Create VNF instance.
+            - Instantiate VNF.
+            - Scale-Out(Will fail).
+            - Get vnfcmOpOccId to retry.
+            - Retry Scale-Out operation.
+            - Get opOccs information.
+            - Terminate VNF.
+            - Delete subscription.
+        """
+        # Create subscription and register it.
+        request_body = fake_vnflcm.Subscription.make_create_request_body(
+            'http://localhost:{}{}'.format(
+                vnflcm_base.FAKE_SERVER_MANAGER.SERVER_PORT,
+                os.path.join(vnflcm_base.MOCK_NOTIFY_CALLBACK_URL,
+                             self._testMethodName)))
+        resp, response_body = self._register_subscription(request_body)
+        self.assertEqual(201, resp.status_code)
+        self.assert_http_header_location_for_subscription(resp.headers)
+        subscription_id = response_body.get('id')
+        self.addCleanup(self._delete_subscription, subscription_id)
+
+        # Pre Setting: Create vnf package.
+        sample_name = 'functional4'
+        csar_package_path = os.path.abspath(
+            os.path.join(
+                os.path.dirname(__file__),
+                "../../../etc/samples/etsi/nfv",
+                sample_name))
+        tempname, _ = vnflcm_base._create_csar_with_unique_vnfd_id(
+            csar_package_path)
+        # upload vnf package
+        vnf_package_id, vnfd_id = vnflcm_base._create_and_upload_vnf_package(
+            self.tacker_client, user_defined_data={
+                "key": sample_name}, temp_csar_path=tempname)
+
+        # Post Setting: Reserve deleting vnf package.
+        self.addCleanup(
+            vnflcm_base._delete_vnf_package,
+            self.tacker_client,
+            vnf_package_id)
+
+        # Create vnf instance
+        resp, vnf_instance = self._create_vnf_instance_from_body(
+            fake_vnflcm.VnfInstances.make_create_request_body(vnfd_id))
+        vnf_instance_id = vnf_instance['id']
+        self._wait_lcm_done(vnf_instance_id=vnf_instance_id)
+        self.assert_create_vnf(resp, vnf_instance, vnf_package_id)
+        self.addCleanup(self._delete_vnf_instance, vnf_instance_id)
+
+        # instantiate VNF
+        request_body = fake_vnflcm.VnfInstances.make_inst_request_body(
+            self.vim['tenant_id'], self.ext_networks, self.ext_mngd_networks,
+            self.ext_link_ports, self.ext_subnets)
+        self._instantiate_vnf_instance(vnf_instance_id, request_body)
+        self._wait_lcm_done('COMPLETED', vnf_instance_id=vnf_instance_id)
+        vnflcm_base.FAKE_SERVER_MANAGER.clear_history(
+            os.path.join(vnflcm_base.MOCK_NOTIFY_CALLBACK_URL,
+                self._testMethodName))
+
+        # Fail Scale-out vnf instance
+        request_body = fake_vnflcm.VnfInstances.make_scale_request_body(
+            'SCALE_OUT')
+        resp, _ = self._scale_vnf_instance(vnf_instance_id, request_body)
+        self._wait_lcm_done('FAILED_TEMP', vnf_instance_id=vnf_instance_id)
+
+        callback_url = os.path.join(
+            vnflcm_base.MOCK_NOTIFY_CALLBACK_URL,
+            self._testMethodName)
+        notify_mock_responses = vnflcm_base.FAKE_SERVER_MANAGER.get_history(
+            callback_url)
+        vnflcm_base.FAKE_SERVER_MANAGER.clear_history(
+            callback_url)
+
+        # get vnflcm_op_occ_id
+        vnflcm_op_occ_id = notify_mock_responses[0].request_body.get(
+            'vnfLcmOpOccId')
+        self.assertIsNotNone(vnflcm_op_occ_id)
+
+        # retry
+        resp, _ = self._retry_op_occs(vnflcm_op_occ_id)
+        self._wait_lcm_done('FAILED_TEMP', vnf_instance_id=vnf_instance_id)
+        self.assert_retry_vnf(resp, vnf_instance_id)
+
+        # rollback (Execute because it's needed to delete VNF)
+        resp, _ = self._rollback_op_occs(vnflcm_op_occ_id)
+        self._wait_lcm_done('ROLLING_BACK', vnf_instance_id=vnf_instance_id)
+        self._wait_lcm_done('ROLLED_BACK', vnf_instance_id=vnf_instance_id)
+        self.assert_rollback_vnf(resp, vnf_instance_id)
+
+        # occ-show
+        resp, op_occs_info = self._show_op_occs(vnflcm_op_occ_id)
+        self._assert_occ_show(resp, op_occs_info)
+
+        # Terminate VNF
+        stack = self._get_heat_stack(vnf_instance_id)
+        resources_list = self._get_heat_resource_list(stack.id)
+        resource_name_list = [r.resource_name for r in resources_list]
+        glance_image_id_list = self._get_glance_image_list_from_stack_resource(
+            stack.id,
+            resource_name_list)
+
+        terminate_req_body = fake_vnflcm.VnfInstances.make_term_request_body()
+        resp, _ = self._terminate_vnf_instance(vnf_instance_id,
+            terminate_req_body)
+        self._wait_lcm_done('COMPLETED', vnf_instance_id=vnf_instance_id)
+        self.assert_terminate_vnf(resp, vnf_instance_id, stack.id,
+            resource_name_list, glance_image_id_list, vnf_package_id)
+
+        # Delete VNF
+        resp, _ = self._delete_vnf_instance(vnf_instance_id)
+        self._wait_lcm_done(vnf_instance_id=vnf_instance_id)
+        self.assert_delete_vnf(resp, vnf_instance_id, vnf_package_id)
+
+        # Subscription delete
+        resp, response_body = self._delete_subscription(subscription_id)
+        self.assertEqual(204, resp.status_code)
+
     def test_rollback_instantiate(self):
         """Test rollback operation for instantiation.
 

tacker/tests/unit/conductor/test_conductor_server.py

@@ -344,6 +344,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnfd_key = 'vnfd_' + instantiate_vnf_req.flavour_id
         vnfd_yaml = vnf_dict['vnfd']['attributes'].get(vnfd_key, '')
         mock_vnfd_dict.return_value = yaml.safe_load(vnfd_yaml)
@@ -387,6 +388,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         instantiate_vnf_req = vnflcm_fakes.get_instantiate_vnf_request_obj()
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         lcm_op_occs_data = fakes.get_lcm_op_occs_data()
         mock_vnf_by_id.return_value = \
@@ -471,6 +473,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         instantiate_vnf_req = vnflcm_fakes.get_instantiate_vnf_request_obj()
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         lcm_op_occs_data = fakes.get_lcm_op_occs_data()
         mock_vnf_by_id.return_value = \
@@ -549,6 +552,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         instantiate_vnf_req = vnflcm_fakes.get_instantiate_vnf_request_obj()
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         lcm_op_occs_data = fakes.get_lcm_op_occs_data()
         vnfd_key = 'vnfd_' + instantiate_vnf_req.flavour_id
@@ -679,6 +683,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         m_vnf_lcm_subscriptions = \
             [mock.MagicMock(**fakes.get_vnf_lcm_subscriptions())]
         vnfd_key = 'vnfd_' + instantiate_vnf_req.flavour_id
@@ -696,6 +701,117 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
             vnf_instance.id, mock.ANY, 'ERROR')
         mock_update_vnf_attributes.assert_called_once()
 
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._update_vnf_attributes')
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._change_vnf_status')
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._build_instantiated_vnf_info')
+    @mock.patch.object(objects.VnfLcmOpOcc, "save")
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.utils._convert_desired_capacity')
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_instantiate_vnf_instance_error_point_notify_processing(
+            self, mock_vnf_by_id, mock_des, mock_vnfd_dict, mock_get_lock,
+            mock_save, mock_build_info, mock_change_vnf_status,
+            mock_update_vnf_attributes):
+        lcm_op_occs_data = fakes.get_lcm_op_occs_data()
+        mock_vnf_by_id.return_value = \
+            objects.VnfLcmOpOcc(context=self.context,
+                                **lcm_op_occs_data)
+
+        vnf_package_vnfd = self._create_and_upload_vnf_package()
+        vnf_instance_data = fake_obj.get_vnf_instance_data(
+            vnf_package_vnfd.vnfd_id)
+        vnf_instance = objects.VnfInstance(context=self.context,
+                                           **vnf_instance_data)
+        vnf_instance.create()
+        instantiate_vnf_req = vnflcm_fakes.get_instantiate_vnf_request_obj()
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
+                                       flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.NOTIFY_PROCESSING
+        vnfd_key = 'vnfd_' + instantiate_vnf_req.flavour_id
+        vnfd_yaml = vnf_dict['vnfd']['attributes'].get(vnfd_key, '')
+        mock_vnfd_dict.return_value = yaml.safe_load(vnfd_yaml)
+        self.conductor.instantiate(self.context, vnf_instance, vnf_dict,
+                                   instantiate_vnf_req, vnf_lcm_op_occs_id)
+        self.vnflcm_driver.instantiate_vnf.assert_called_once_with(
+            self.context, mock.ANY, vnf_dict, instantiate_vnf_req)
+        self.vnflcm_driver._vnf_instance_update.assert_called_once()
+        mock_change_vnf_status. \
+            assert_called_once_with(self.context, vnf_instance.id,
+                                    mock.ANY, 'PENDING_CREATE')
+        mock_update_vnf_attributes.assert_called_once()
+
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._update_vnf_attributes')
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._build_instantiated_vnf_info')
+    @mock.patch.object(objects.VnfLcmOpOcc, "save")
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.utils._convert_desired_capacity')
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_instantiate_vnf_instance_error_point_vnf_config_end(
+            self, mock_vnf_by_id, mock_des, mock_vnfd_dict, mock_get_lock,
+            mock_save, mock_build_info, mock_update_vnf_attributes):
+        lcm_op_occs_data = fakes.get_lcm_op_occs_data()
+        mock_vnf_by_id.return_value = \
+            objects.VnfLcmOpOcc(context=self.context,
+                                **lcm_op_occs_data)
+
+        vnf_package_vnfd = self._create_and_upload_vnf_package()
+        vnf_instance_data = fake_obj.get_vnf_instance_data(
+            vnf_package_vnfd.vnfd_id)
+        vnf_instance = objects.VnfInstance(context=self.context,
+                                           **vnf_instance_data)
+        vnf_instance.create()
+        instantiate_vnf_req = vnflcm_fakes.get_instantiate_vnf_request_obj()
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
+                                       flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.VNF_CONFIG_END
+        vnfd_key = 'vnfd_' + instantiate_vnf_req.flavour_id
+        vnfd_yaml = vnf_dict['vnfd']['attributes'].get(vnfd_key, '')
+        mock_vnfd_dict.return_value = yaml.safe_load(vnfd_yaml)
+        self.conductor.instantiate(self.context, vnf_instance, vnf_dict,
+                                   instantiate_vnf_req, vnf_lcm_op_occs_id)
+        self.vnflcm_driver.instantiate_vnf.assert_called_once_with(
+            self.context, vnf_instance, vnf_dict, instantiate_vnf_req)
+
+    @mock.patch.object(objects.VnfLcmOpOcc, "save")
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.utils._convert_desired_capacity')
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_instantiate_vnf_instance_error_point_notify_completed(
+            self, mock_vnf_by_id, mock_des, mock_vnfd_dict,
+            mock_get_lock, mock_save):
+        lcm_op_occs_data = fakes.get_lcm_op_occs_data()
+        mock_vnf_by_id.return_value = \
+            objects.VnfLcmOpOcc(context=self.context,
+                                **lcm_op_occs_data)
+
+        vnf_package_vnfd = self._create_and_upload_vnf_package()
+        vnf_instance_data = fake_obj.get_vnf_instance_data(
+            vnf_package_vnfd.vnfd_id)
+        vnf_instance = objects.VnfInstance(context=self.context,
+                                           **vnf_instance_data)
+        vnf_instance.create()
+        instantiate_vnf_req = vnflcm_fakes.get_instantiate_vnf_request_obj()
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
+                                       flavour=instantiate_vnf_req.flavour_id)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.NOTIFY_COMPLETED
+        vnfd_key = 'vnfd_' + instantiate_vnf_req.flavour_id
+        vnfd_yaml = vnf_dict['vnfd']['attributes'].get(vnfd_key, '')
+        mock_vnfd_dict.return_value = yaml.safe_load(vnfd_yaml)
+        self.conductor.instantiate(self.context, vnf_instance, vnf_dict,
+                                   instantiate_vnf_req, vnf_lcm_op_occs_id)
+        self.vnflcm_driver._vnf_instance_update.assert_called_once()
+
     @mock.patch('tacker.conductor.conductor_server.Conductor'
                 '._change_vnf_status')
     @mock.patch('tacker.conductor.conductor_server.Conductor'
@@ -713,15 +829,98 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
             additional_params={"key": "value"})
         vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
         vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         self.conductor.terminate(self.context, vnf_lcm_op_occs_id,
                                  vnf_instance, terminate_vnf_req, vnf_dict)
 
         self.vnflcm_driver.terminate_vnf.assert_called_once_with(
-            self.context, vnf_instance, terminate_vnf_req)
+            self.context, vnf_instance, terminate_vnf_req, vnf_dict)
         self.vnflcm_driver._vnf_instance_update.assert_called_once()
         self.assertEqual(mock_send_notification.call_count, 2)
         self.assertEqual(mock_change_vnf_status.call_count, 2)
 
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._change_vnf_status')
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._send_lcm_op_occ_notification')
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    def test_terminate_vnf_instance_error_point_notify_processing(
+            self, mock_get_lock, mock_send_notification,
+            mock_change_vnf_status):
+        inst_vnf_info = fd_utils.get_vnf_instantiated_info()
+        vnf_instance = fd_utils. \
+            get_vnf_instance_object(instantiated_vnf_info=inst_vnf_info)
+
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.GRACEFUL,
+            additional_params={"key": "value"})
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.NOTIFY_PROCESSING
+        self.conductor.terminate(self.context, vnf_lcm_op_occs_id,
+                                 vnf_instance, terminate_vnf_req, vnf_dict)
+
+        self.vnflcm_driver.terminate_vnf.assert_called_once_with(
+            self.context, vnf_instance, terminate_vnf_req, vnf_dict)
+        self.vnflcm_driver._vnf_instance_update.assert_called_once()
+        self.assertEqual(mock_send_notification.call_count, 2)
+        self.assertEqual(mock_change_vnf_status.call_count, 2)
+
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._change_vnf_status')
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._send_lcm_op_occ_notification')
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    def test_terminate_vnf_instance_error_point_internal_processing(
+            self, mock_get_lock, mock_send_notification,
+            mock_change_vnf_status):
+        inst_vnf_info = fd_utils.get_vnf_instantiated_info()
+        vnf_instance = fd_utils. \
+            get_vnf_instance_object(instantiated_vnf_info=inst_vnf_info)
+
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.GRACEFUL,
+            additional_params={"key": "value"})
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INTERNAL_PROCESSING
+        self.conductor.terminate(self.context, vnf_lcm_op_occs_id,
+                                 vnf_instance, terminate_vnf_req, vnf_dict)
+
+        self.vnflcm_driver.terminate_vnf.assert_called_once_with(
+            self.context, vnf_instance, terminate_vnf_req, vnf_dict)
+        self.vnflcm_driver._vnf_instance_update.assert_called_once()
+        self.assertEqual(mock_send_notification.call_count, 2)
+        self.assertEqual(mock_change_vnf_status.call_count, 1)
+
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver'
+                '.terminate_vnf')
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._change_vnf_status')
+    @mock.patch('tacker.conductor.conductor_server.Conductor'
+                '._send_lcm_op_occ_notification')
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    def test_terminate_vnf_instance_error_point_notify_completed(
+            self, mock_get_lock, mock_send_notification,
+            mock_change_vnf_status, mock_vnflcm_driver_terminate_vnf):
+        inst_vnf_info = fd_utils.get_vnf_instantiated_info()
+        vnf_instance = fd_utils. \
+            get_vnf_instance_object(instantiated_vnf_info=inst_vnf_info)
+
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.GRACEFUL,
+            additional_params={"key": "value"})
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.NOTIFY_COMPLETED
+        self.conductor.terminate(self.context, vnf_lcm_op_occs_id,
+                                 vnf_instance, terminate_vnf_req, vnf_dict)
+
+        self.assertEqual(mock_vnflcm_driver_terminate_vnf.call_count, 0)
+        self.vnflcm_driver._vnf_instance_update.assert_called_once()
+        self.assertEqual(mock_send_notification.call_count, 2)
+        self.assertEqual(mock_change_vnf_status.call_count, 0)
+
     @mock.patch('tacker.conductor.conductor_server.Conductor'
                 '._change_vnf_status')
     @mock.patch.object(coordination.Coordinator, 'get_lock')
@@ -751,6 +950,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
             termination_type=fields.VnfInstanceTerminationType.GRACEFUL)
         vnfLcmOpOccId = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.VNF_CONFIG_END
 
         mock_exec.return_value = True
         resRemResource = []
@@ -766,7 +966,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
                                 vnf_instance, terminate_vnf_req, vnf_dict)
 
         self.vnflcm_driver.terminate_vnf.assert_called_once_with(
-            self.context, mock.ANY, terminate_vnf_req)
+            self.context, mock.ANY, terminate_vnf_req, vnf_dict)
 
     @mock.patch('tacker.conductor.conductor_server.Conductor'
                 '._change_vnf_status')
@@ -860,6 +1060,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
             termination_type=fields.VnfInstanceTerminationType.GRACEFUL)
         vnfLcmOpOccId = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.VNF_CONFIG_END
 
         mock_exec.return_value = True
         grant_dict = {}
@@ -874,7 +1075,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
                                 vnf_instance, terminate_vnf_req, vnf_dict)
 
         self.vnflcm_driver.terminate_vnf.assert_called_once_with(
-            self.context, mock.ANY, terminate_vnf_req)
+            self.context, mock.ANY, terminate_vnf_req, vnf_dict)
 
     @mock.patch('tacker.conductor.conductor_server.Conductor'
                 '.send_notification')
@@ -908,6 +1109,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
             termination_type=fields.VnfInstanceTerminationType.GRACEFUL)
         vnfLcmOpOccId = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
 
         mock_exec.return_value = True
         resRemResource = []
@@ -965,6 +1167,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
             termination_type=fields.VnfInstanceTerminationType.GRACEFUL)
         vnfLcmOpOccId = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
 
         mock_exec.return_value = True
         mock_grants.side_effect = \
@@ -995,6 +1198,8 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
             additional_params={"key": "value"})
         vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
         vnf_dict = db_utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
+        vnf_dict['current_error_point'] = fields.ErrorPoint.INITIAL
         try:
             self.conductor.terminate(self.context, vnf_lcm_op_occs_id,
                                  vnf_instance, terminate_vnf_req, vnf_dict)
@@ -1148,8 +1353,8 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
     @mock.patch.object(objects.VnfLcmOpOcc, "save")
     @mock.patch.object(coordination.Coordinator, 'get_lock')
     @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
-    def test_heal_vnf_instance(self, mock_vnf_by_id, mock_get_lock,
+    def test_heal_vnf_instance(self, mock_vnf_by_id,
-            mock_save, mock_change_vnf_status,
+            mock_get_lock, mock_save, mock_change_vnf_status,
             mock_update_insta_vnf_info, mock_add_additional_vnf_info):
         lcm_op_occs_data = fakes.get_lcm_op_occs_data()
         mock_vnf_by_id.return_value = \
@@ -1166,6 +1371,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_instance.save()
         heal_vnf_req = objects.HealVnfRequest(cause="healing request")
         vnf_dict = {"fake": "fake_dict"}
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
         self.conductor.heal(self.context, vnf_instance, vnf_dict,
                             heal_vnf_req, vnf_lcm_op_occs_id)
@@ -1175,6 +1381,121 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         mock_add_additional_vnf_info. \
             assert_called_once_with(self.context, vnf_instance)
 
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_add_additional_vnf_info')
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_update_instantiated_vnf_info')
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_change_vnf_status')
+    @mock.patch.object(objects.VnfLcmOpOcc, "save")
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_heal_vnf_instance_error_point_notify_processing(
+            self, mock_vnf_by_id, mock_get_lock, mock_save,
+            mock_change_vnf_status, mock_update_insta_vnf_info,
+            mock_add_additional_vnf_info):
+        lcm_op_occs_data = fakes.get_lcm_op_occs_data()
+        mock_vnf_by_id.return_value = \
+            objects.VnfLcmOpOcc(context=self.context,
+                                **lcm_op_occs_data)
+        vnf_package_vnfd = self._create_and_upload_vnf_package()
+        vnf_instance_data = fake_obj.get_vnf_instance_data(
+            vnf_package_vnfd.vnfd_id)
+        vnf_instance = objects.VnfInstance(context=self.context,
+                                           **vnf_instance_data)
+        vnf_instance.create()
+        vnf_instance.instantiation_state = \
+            fields.VnfInstanceState.INSTANTIATED
+        vnf_instance.save()
+        heal_vnf_req = objects.HealVnfRequest(cause="healing request")
+        vnf_dict = {"fake": "fake_dict"}
+        vnf_dict['before_error_point'] = fields.ErrorPoint.NOTIFY_PROCESSING
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        self.conductor.heal(self.context, vnf_instance, vnf_dict,
+                            heal_vnf_req, vnf_lcm_op_occs_id)
+        self.assertEqual(mock_change_vnf_status.call_count, 2)
+        mock_update_insta_vnf_info. \
+            assert_called_once_with(self.context, vnf_instance, heal_vnf_req)
+        mock_add_additional_vnf_info. \
+            assert_called_once_with(self.context, vnf_instance)
+
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_add_additional_vnf_info')
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_update_instantiated_vnf_info')
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_change_vnf_status')
+    @mock.patch.object(objects.VnfLcmOpOcc, "save")
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_heal_vnf_instance_error_point_internal_processing(
+            self, mock_vnf_by_id, mock_get_lock, mock_save,
+            mock_change_vnf_status, mock_update_insta_vnf_info,
+            mock_add_additional_vnf_info):
+        lcm_op_occs_data = fakes.get_lcm_op_occs_data()
+        mock_vnf_by_id.return_value = \
+            objects.VnfLcmOpOcc(context=self.context,
+                                **lcm_op_occs_data)
+        vnf_package_vnfd = self._create_and_upload_vnf_package()
+        vnf_instance_data = fake_obj.get_vnf_instance_data(
+            vnf_package_vnfd.vnfd_id)
+        vnf_instance = objects.VnfInstance(context=self.context,
+                                           **vnf_instance_data)
+        vnf_instance.create()
+        vnf_instance.instantiation_state = \
+            fields.VnfInstanceState.INSTANTIATED
+        vnf_instance.save()
+        heal_vnf_req = objects.HealVnfRequest(cause="healing request")
+        vnf_dict = {"fake": "fake_dict"}
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INTERNAL_PROCESSING
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        self.conductor.heal(self.context, vnf_instance, vnf_dict,
+                            heal_vnf_req, vnf_lcm_op_occs_id)
+        self.assertEqual(mock_change_vnf_status.call_count, 1)
+        mock_update_insta_vnf_info. \
+            assert_called_once_with(self.context, vnf_instance, heal_vnf_req)
+        mock_add_additional_vnf_info. \
+            assert_called_once_with(self.context, vnf_instance)
+
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver'
+                '.heal_vnf')
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_add_additional_vnf_info')
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_update_instantiated_vnf_info')
+    @mock.patch('tacker.conductor.conductor_server.Conductor.'
+                '_change_vnf_status')
+    @mock.patch.object(objects.VnfLcmOpOcc, "save")
+    @mock.patch.object(coordination.Coordinator, 'get_lock')
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_heal_vnf_instance_error_point_notify_completed(
+            self, mock_vnf_by_id, mock_get_lock, mock_save,
+            mock_change_vnf_status, mock_update_insta_vnf_info,
+            mock_add_additional_vnf_info, mock_vnflcm_driver_heal_vnf):
+        lcm_op_occs_data = fakes.get_lcm_op_occs_data()
+        mock_vnf_by_id.return_value = \
+            objects.VnfLcmOpOcc(context=self.context,
+                                **lcm_op_occs_data)
+        vnf_package_vnfd = self._create_and_upload_vnf_package()
+        vnf_instance_data = fake_obj.get_vnf_instance_data(
+            vnf_package_vnfd.vnfd_id)
+        vnf_instance = objects.VnfInstance(context=self.context,
+                                           **vnf_instance_data)
+        vnf_instance.create()
+        vnf_instance.instantiation_state = \
+            fields.VnfInstanceState.INSTANTIATED
+        vnf_instance.save()
+        heal_vnf_req = objects.HealVnfRequest(cause="healing request")
+        vnf_dict = {"fake": "fake_dict"}
+        vnf_dict['before_error_point'] = fields.ErrorPoint.NOTIFY_COMPLETED
+        vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
+        self.conductor.heal(self.context, vnf_instance, vnf_dict,
+                            heal_vnf_req, vnf_lcm_op_occs_id)
+        self.assertEqual(mock_vnflcm_driver_heal_vnf.call_count, 0)
+        self.assertEqual(mock_change_vnf_status.call_count, 0)
+        self.assertEqual(mock_update_insta_vnf_info.call_count, 0)
+        self.assertEqual(mock_add_additional_vnf_info.call_count, 0)
+
     @mock.patch('tacker.conductor.conductor_server.Conductor'
                 '._change_vnf_status')
     @mock.patch('tacker.conductor.conductor_server.Conductor'
@@ -1222,6 +1543,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         heal_vnf_req = objects.HealVnfRequest(cause="healing request")
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour='simple')
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         mock_exec.return_value = True
         mock_act.return_value = None
@@ -1355,6 +1677,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         heal_vnf_req = objects.HealVnfRequest(cause="healing request")
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour='simple')
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         mock_exec.return_value = True
         mock_act.return_value = None
@@ -1473,6 +1796,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         heal_vnf_req = objects.HealVnfRequest(cause="healing request")
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour='simple')
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         mock_exec.return_value = True
         mock_act.return_value = None
@@ -1566,6 +1890,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         heal_vnf_req = objects.HealVnfRequest(cause="healing request")
         vnf_dict = db_utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
                                        flavour='simple')
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = 'a9c36d21-21aa-4692-8922-7999bbcae08c'
         mock_exec.return_value = True
         mock_act.return_value = None
@@ -1618,11 +1943,12 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
 
         heal_vnf_req = objects.HealVnfRequest(cause="healing request")
         vnf_dict = {"fake": "fake_dict"}
+        vnf_dict['before_error_point'] = fields.ErrorPoint.INITIAL
         vnf_lcm_op_occs_id = uuidsentinel.vnf_lcm_op_occs_id
         self.conductor.heal(self.context, vnf_instance, vnf_dict,
                             heal_vnf_req, vnf_lcm_op_occs_id)
         mock_change_vnf_status.assert_called_with(self.context,
-            vnf_instance, mock.ANY, constants.ERROR, "")
+            vnf_instance.id, mock.ANY, constants.ERROR, "")
         mock_update_insta_vnf_info.assert_called_with(self.context,
             vnf_instance, heal_vnf_req)
         self.assertEqual(mock_send_notification.call_count, 2)
@@ -1770,6 +2096,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_info['removeResources'] = []
         vnf_info['affinity_list'] = []
         vnf_info['placement_constraint_list'] = []
+        vnf_info['before_error_point'] = fields.ErrorPoint.INITIAL
         grant_dict = {}
         grant_dict['id'] = 'c213e465-8220-487e-9464-f79104e81e96'
         grant_dict['vnf_instance_id'] = uuidsentinel.vnf_instance_id
@@ -1886,6 +2213,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_info['removeResources'] = removeResources
         vnf_info['affinity_list'] = []
         vnf_info['placement_constraint_list'] = []
+        vnf_info['before_error_point'] = fields.ErrorPoint.INITIAL
         grant_dict = {}
         grant_dict['id'] = 'c213e465-8220-487e-9464-f79104e81e96'
         grant_dict['vnfInstanceId'] = uuidsentinel.vnf_instance_id
@@ -2030,6 +2358,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_info['affinity_list'] = []
         vnf_info['placement_constraint_list'] = []
         vnf_info['placement_constraint_list'].append(placement)
+        vnf_info['before_error_point'] = fields.ErrorPoint.INITIAL
         grant_dict = {}
         grant_dict['id'] = 'c213e465-8220-487e-9464-f79104e81e96'
         grant_dict['vnfInstanceId'] = uuidsentinel.vnf_instance_id
@@ -2138,6 +2467,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_info['removeResources'] = []
         vnf_info['affinity_list'] = []
         vnf_info['placement_constraint_list'] = []
+        vnf_info['before_error_point'] = fields.ErrorPoint.INITIAL
         grant_dict = {}
         grant_dict['id'] = 'c213e465-8220-487e-9464-f79104e81e96'
         grant_dict['vnf_instance_id'] = uuidsentinel.vnf_instance_id
@@ -2239,6 +2569,7 @@ class TestConductor(SqlTestCase, unit_base.FixturedTestCase):
         vnf_info['removeResources'] = []
         vnf_info['affinity_list'] = []
         vnf_info['placement_constraint_list'] = []
+        vnf_info['before_error_point'] = fields.ErrorPoint.INITIAL
         moch_exec.return_value = True
         mock_grants.side_effect = \
             requests.exceptions.HTTPError("MockException")

tacker/tests/unit/vnflcm/fakes.py

@@ -1460,3 +1460,27 @@ def return_vnf_lcm_opoccs_obj():
     obj = objects.VnfLcmOpOcc(**vnf_lcm_op_occs)
 
     return obj
+
+
+def vnflcm_op_occs_retry_data(error_point=7, operation='INSTANTIATE',
+        operation_state='FAILED_TEMP'):
+    now = datetime.datetime(2000, 1, 1, 1, 1, 1, tzinfo=iso8601.UTC)
+    return objects.VnfLcmOpOcc(
+        state_entered_time=now,
+        start_time=now,
+        vnf_instance_id=uuidsentinel.vnf_instance_id,
+        operation=operation,
+        operation_state=operation_state,
+        is_automatic_invocation=False,
+        operation_params='{}',
+        error_point=error_point,
+        id=constants.UUID,
+        created_at=now)
+
+
+def vnf_data(status='ACTIVE'):
+    return tacker.db.vnfm.vnfm_db.VNF(id=constants.UUID,
+        vnfd_id=uuidsentinel.vnfd_id,
+        name='test',
+        status=status,
+        vim_id=uuidsentinel.vim_id)

tacker/tests/unit/vnflcm/test_controller.py

@@ -1359,32 +1359,6 @@ class TestController(base.TestCase):
                          resp.json['itemNotFound']['message'])
         mock_get_vnf.assert_called_once()
 
-    @mock.patch.object(TackerManager, 'get_service_plugins',
-                       return_value={'VNFM':
-                       test_nfvo_plugin.FakeVNFMPlugin()})
-    @mock.patch('tacker.api.vnflcm.v1.controller.'
-                'VnfLcmController._get_vnf')
-    @mock.patch.object(objects.vnf_instance, "_vnf_instance_get_by_id")
-    def test_terminate_incorrect_instantiation_state(
-            self, mock_vnf_by_id, mock_get_vnf, mock_get_service_plugins):
-        mock_vnf_by_id.return_value = fakes.return_vnf_instance()
-        body = {"terminationType": "FORCEFUL"}
-        req = fake_request.HTTPRequest.blank(
-            '/vnf_instances/%s/terminate' % uuidsentinel.vnf_instance_id)
-        req.body = jsonutils.dump_as_bytes(body)
-        req.headers['Content-Type'] = 'application/json'
-        req.method = 'POST'
-
-        resp = req.get_response(self.app)
-
-        self.assertEqual(http_client.CONFLICT, resp.status_code)
-        expected_msg = ("Vnf instance %s in instantiation_state "
-                        "NOT_INSTANTIATED. Cannot terminate while the vnf "
-                        "instance is in this state.")
-        self.assertEqual(expected_msg % uuidsentinel.vnf_instance_id,
-                         resp.json['conflictingRequest']['message'])
-        mock_get_vnf.assert_called_once()
-
     @mock.patch.object(TackerManager, 'get_service_plugins',
                        return_value={'VNFM':
                        test_nfvo_plugin.FakeVNFMPlugin()})
@@ -3130,3 +3104,265 @@ class TestController(base.TestCase):
         res_dict = self.controller.fail(req, constants.UUID)
 
         self.assertEqual(http_client.CONFLICT, res_dict.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    @mock.patch.object(objects.VnfInstance, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_instantiate")
+    def test_retry_instantiate_vnf_instance(
+            self, mock_instantiate, mock_vnf_instance, mock_get_vnf,
+            mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data()
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+        vnf_obj = fakes.vnf_data()
+        mock_get_vnf.return_value = vnf_obj
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        mock_vnf_instance.return_value = vnf_instance
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.ACCEPTED, resp.status_code)
+        mock_instantiate.assert_called_once()
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    @mock.patch.object(objects.VnfInstance, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_terminate")
+    def test_retry_terminate_vnf_instance(
+            self, mock_terminate, mock_vnf_instance, mock_get_vnf,
+            mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data(
+            operation='TERMINATE')
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+        vnf_obj = fakes.vnf_data()
+        mock_get_vnf.return_value = vnf_obj
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        mock_vnf_instance.return_value = vnf_instance
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.ACCEPTED, resp.status_code)
+        mock_terminate.assert_called_once()
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    @mock.patch.object(objects.VnfInstance, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_heal")
+    def test_retry_heal_vnf_instance(
+            self, mock_heal, mock_vnf_instance, mock_get_vnf, mock_lcm_by_id,
+            mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data(operation='HEAL')
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+        vnf_obj = fakes.vnf_data()
+        mock_get_vnf.return_value = vnf_obj
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        mock_vnf_instance.return_value = vnf_instance
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.ACCEPTED, resp.status_code)
+        mock_heal.assert_called_once()
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    @mock.patch.object(objects.VnfInstance, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_scale")
+    def test_retry_scale_vnf_instance(
+            self, mock_scale, mock_vnf_instance, mock_get_vnf, mock_lcm_by_id,
+            mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data(operation='SCALE')
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+        vnf_obj = fakes.vnf_data()
+        mock_get_vnf.return_value = vnf_obj
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        mock_vnf_instance.return_value = vnf_instance
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.ACCEPTED, resp.status_code)
+        mock_scale.assert_called_once()
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_retry_vnf_lcm_occ_not_found(
+            self, mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        mock_lcm_by_id.side_effect = exceptions.NotFound
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.NOT_FOUND, resp.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_retry_vnf_lcm_occ_error(
+            self, mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        mock_lcm_by_id.side_effect = Exception
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.INTERNAL_SERVER_ERROR, resp.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    def test_retry_vnf_lcm_occ_conflict(
+            self, mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data(
+            operation_state='invalid operation state')
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.CONFLICT, resp.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    def test_retry_vnf_not_found(
+            self, mock_get_vnf, mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data()
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+
+        mock_get_vnf.side_effect = exc.HTTPNotFound
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.NOT_FOUND, resp.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    def test_retry_vnf_error(
+            self, mock_get_vnf, mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data()
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+
+        mock_get_vnf.side_effect = exc.HTTPInternalServerError
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.INTERNAL_SERVER_ERROR, resp.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    @mock.patch.object(objects.VnfInstance, "get_by_id")
+    def test_retry_vnf_instance_not_found(
+            self, mock_get_vnf_instance, mock_get_vnf,
+            mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data()
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+        vnf_obj = fakes.vnf_data()
+        mock_get_vnf.return_value = vnf_obj
+
+        mock_get_vnf_instance.side_effect = exceptions.VnfInstanceNotFound
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.NOT_FOUND, resp.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    @mock.patch.object(objects.VnfInstance, "get_by_id")
+    def test_retry_vnf_instance_error(
+            self, mock_get_vnf_instance, mock_get_vnf,
+            mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data()
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+        vnf_obj = fakes.vnf_data()
+        mock_get_vnf.return_value = vnf_obj
+
+        mock_get_vnf_instance.side_effect = Exception
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.INTERNAL_SERVER_ERROR, resp.status_code)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_id")
+    @mock.patch.object(controller.VnfLcmController, "_get_vnf")
+    @mock.patch.object(objects.VnfInstance, "get_by_id")
+    def test_retry_invalid_operation_type(
+            self, mock_get_vnf_instance, mock_get_vnf,
+            mock_lcm_by_id, mock_get_service_plugins):
+        req = fake_request.HTTPRequest.blank(
+            '/vnf_lcm_op_occs/%s/retry' % uuidsentinel.vnf_lcm_op_occs_id)
+        req.headers['Content-Type'] = 'application/json'
+        req.method = 'POST'
+
+        vnf_lcm_op_occs = fakes.vnflcm_op_occs_retry_data(
+            operation='Invalid operation type')
+        mock_lcm_by_id.return_value = vnf_lcm_op_occs
+        vnf_obj = fakes.vnf_data()
+        mock_get_vnf.return_value = vnf_obj
+
+        resp = req.get_response(self.app)
+        self.assertEqual(http_client.INTERNAL_SERVER_ERROR, resp.status_code)

tacker/tests/unit/vnflcm/test_load_vnf_interfaces.py

@@ -25,6 +25,7 @@ from tacker.common import exceptions
 from tacker import context
 from tacker.manager import TackerManager
 from tacker import objects
+from tacker.objects import fields
 from tacker.tests.unit.db import base as db_base
 from tacker.tests.unit.nfvo.test_nfvo_plugin import FakeVNFMPlugin
 from tacker.tests.unit.vnflcm import fakes
@@ -165,7 +166,9 @@ class MgmtVnfLcmDriverTest(db_base.SqlTestCase):
         test_utils.copy_csar_files(fake_csar, "refactor_mgmt_driver1")
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": fields.ErrorPoint.VNF_CONFIG_START}
         driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
                                instantiate_vnf_req_obj)
 
@@ -203,7 +206,9 @@ class MgmtVnfLcmDriverTest(db_base.SqlTestCase):
         test_utils.copy_csar_files(fake_csar, "refactor_mgmt_driver2")
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": fields.ErrorPoint.VNF_CONFIG_START}
         self.assertRaises(exceptions.MgmtDriverInconsistent,
                           driver.instantiate_vnf, self.context,
                           vnf_instance_obj, vnf_dict,
@@ -240,7 +245,9 @@ class MgmtVnfLcmDriverTest(db_base.SqlTestCase):
         test_utils.copy_csar_files(fake_csar, "refactor_mgmt_driver3")
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": fields.ErrorPoint.VNF_CONFIG_START}
         self.assertRaises(exceptions.MgmtDriverHashMatchFailure,
                           driver.instantiate_vnf, self.context,
                           vnf_instance_obj, vnf_dict,

tacker/tests/unit/vnflcm/test_vnflcm_driver.py

@@ -33,6 +33,7 @@ from tacker.db.common_services import common_services_db_plugin
 from tacker.manager import TackerManager
 from tacker import objects
 from tacker.objects import fields
+from tacker.objects.fields import ErrorPoint as EP
 from tacker.objects import vim_connection
 from tacker.tests.unit.db import base as db_base
 from tacker.tests.unit.nfvo.test_nfvo_plugin import FakeVNFMPlugin
@@ -216,7 +217,9 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         test_utils.copy_csar_files(fake_csar, "vnflcm4")
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL}
         driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
                                instantiate_vnf_req_obj)
 
@@ -224,6 +227,231 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         self.assertEqual(5, self._vnf_manager.invoke.call_count)
         shutil.rmtree(fake_csar)
 
+    @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfResource, 'create')
+    @mock.patch.object(objects.VnfPackageVnfd, 'get_by_id')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_instantiate_vnf_with_error_point_vnf_config_start(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_vnf_instance_save, mock_vnf_package_vnfd, mock_create,
+            mock_get_service_plugins, mock_init_hash, mock_final_vnf_dict):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_package_vnfd = fakes.return_vnf_package_vnfd()
+        vnf_package_id = vnf_package_vnfd.package_uuid
+        mock_vnf_package_vnfd.return_value = vnf_package_vnfd
+        instantiate_vnf_req_dict = fakes.get_dummy_instantiate_vnf_request()
+        instantiate_vnf_req_obj = \
+            objects.InstantiateVnfRequest.obj_from_primitive(
+                instantiate_vnf_req_dict, self.context)
+        vnf_instance_obj = fakes.return_vnf_instance()
+
+        fake_csar = os.path.join(self.temp_dir, vnf_package_id)
+        cfg.CONF.set_override('vnf_package_csar_path', self.temp_dir,
+                              group='vnf_package')
+        test_utils.copy_csar_files(fake_csar, "vnflcm4")
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.VNF_CONFIG_START}
+        driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
+                               instantiate_vnf_req_obj)
+
+        self.assertEqual(1, mock_vnf_instance_save.call_count)
+        self.assertEqual(5, self._vnf_manager.invoke.call_count)
+        shutil.rmtree(fake_csar)
+
+    @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfResource, 'create')
+    @mock.patch.object(objects.VnfPackageVnfd, 'get_by_id')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_instantiate_vnf_with_error_point_pre_vim_control(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_vnf_instance_save, mock_vnf_package_vnfd, mock_create,
+            mock_get_service_plugins, mock_init_hash, mock_final_vnf_dict):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_package_vnfd = fakes.return_vnf_package_vnfd()
+        vnf_package_id = vnf_package_vnfd.package_uuid
+        mock_vnf_package_vnfd.return_value = vnf_package_vnfd
+        instantiate_vnf_req_dict = fakes.get_dummy_instantiate_vnf_request()
+        instantiate_vnf_req_obj = \
+            objects.InstantiateVnfRequest.obj_from_primitive(
+                instantiate_vnf_req_dict, self.context)
+        vnf_instance_obj = fakes.return_vnf_instance()
+
+        fake_csar = os.path.join(self.temp_dir, vnf_package_id)
+        cfg.CONF.set_override('vnf_package_csar_path', self.temp_dir,
+                              group='vnf_package')
+        test_utils.copy_csar_files(fake_csar, "vnflcm4")
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.PRE_VIM_CONTROL}
+        driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
+                               instantiate_vnf_req_obj)
+
+        self.assertEqual(1, mock_vnf_instance_save.call_count)
+        self.assertEqual(4, self._vnf_manager.invoke.call_count)
+        shutil.rmtree(fake_csar)
+
+    @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfResource, 'create')
+    @mock.patch.object(objects.VnfPackageVnfd, 'get_by_id')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_instantiate_vnf_with_error_point_post_vim_control(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_vnf_instance_save, mock_vnf_package_vnfd, mock_create,
+            mock_get_service_plugins, mock_init_hash, mock_final_vnf_dict):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_package_vnfd = fakes.return_vnf_package_vnfd()
+        vnf_package_id = vnf_package_vnfd.package_uuid
+        mock_vnf_package_vnfd.return_value = vnf_package_vnfd
+        instantiate_vnf_req_dict = fakes.get_dummy_instantiate_vnf_request()
+        instantiate_vnf_req_obj = \
+            objects.InstantiateVnfRequest.obj_from_primitive(
+                instantiate_vnf_req_dict, self.context)
+        vnf_instance_obj = fakes.return_vnf_instance()
+
+        fake_csar = os.path.join(self.temp_dir, vnf_package_id)
+        cfg.CONF.set_override('vnf_package_csar_path', self.temp_dir,
+                              group='vnf_package')
+        test_utils.copy_csar_files(fake_csar, "vnflcm4")
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.POST_VIM_CONTROL}
+        driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
+                               instantiate_vnf_req_obj)
+
+        self.assertEqual(1, mock_vnf_instance_save.call_count)
+        self.assertEqual(4, self._vnf_manager.invoke.call_count)
+        shutil.rmtree(fake_csar)
+
+    @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfResource, 'create')
+    @mock.patch.object(objects.VnfPackageVnfd, 'get_by_id')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_instantiate_vnf_with_error_point_internal_processing(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_vnf_instance_save, mock_vnf_package_vnfd, mock_create,
+            mock_get_service_plugins, mock_init_hash, mock_final_vnf_dict):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_package_vnfd = fakes.return_vnf_package_vnfd()
+        vnf_package_id = vnf_package_vnfd.package_uuid
+        mock_vnf_package_vnfd.return_value = vnf_package_vnfd
+        instantiate_vnf_req_dict = fakes.get_dummy_instantiate_vnf_request()
+        instantiate_vnf_req_obj = \
+            objects.InstantiateVnfRequest.obj_from_primitive(
+                instantiate_vnf_req_dict, self.context)
+        vnf_instance_obj = fakes.return_vnf_instance()
+
+        fake_csar = os.path.join(self.temp_dir, vnf_package_id)
+        cfg.CONF.set_override('vnf_package_csar_path', self.temp_dir,
+                              group='vnf_package')
+        test_utils.copy_csar_files(fake_csar, "vnflcm4")
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INTERNAL_PROCESSING}
+        driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
+                               instantiate_vnf_req_obj)
+
+        self.assertEqual(1, mock_vnf_instance_save.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
+        shutil.rmtree(fake_csar)
+
+    @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(objects.VnfResource, 'create')
+    @mock.patch.object(objects.VnfPackageVnfd, 'get_by_id')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_instantiate_vnf_with_error_point_vnf_config_end(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_vnf_instance_save, mock_vnf_package_vnfd, mock_create,
+            mock_get_service_plugins, mock_init_hash, mock_final_vnf_dict):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_package_vnfd = fakes.return_vnf_package_vnfd()
+        vnf_package_id = vnf_package_vnfd.package_uuid
+        mock_vnf_package_vnfd.return_value = vnf_package_vnfd
+        instantiate_vnf_req_dict = fakes.get_dummy_instantiate_vnf_request()
+        instantiate_vnf_req_obj = \
+            objects.InstantiateVnfRequest.obj_from_primitive(
+                instantiate_vnf_req_dict, self.context)
+        vnf_instance_obj = fakes.return_vnf_instance()
+
+        fake_csar = os.path.join(self.temp_dir, vnf_package_id)
+        cfg.CONF.set_override('vnf_package_csar_path', self.temp_dir,
+                              group='vnf_package')
+        test_utils.copy_csar_files(fake_csar, "vnflcm4")
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.VNF_CONFIG_END}
+        driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
+                               instantiate_vnf_req_obj)
+
+        self.assertEqual(1, mock_vnf_instance_save.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
+        shutil.rmtree(fake_csar)
+
     @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
     @mock.patch.object(VnfLcmDriver,
                        '_init_mgmt_driver_hash')
@@ -261,7 +489,9 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         test_utils.copy_csar_files(fake_csar, "vnflcm4")
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL}
         driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
                                instantiate_vnf_req_obj)
 
@@ -306,7 +536,9 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         test_utils.copy_csar_files(fake_csar, "vnflcm4")
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL}
         driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
                                instantiate_vnf_req_obj)
 
@@ -351,7 +583,9 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         test_utils.copy_csar_files(fake_csar, "vnflcm4")
         self._mock_vnf_manager(fail_method_name="instantiate_vnf")
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL}
         error = self.assertRaises(exceptions.VnfInstantiationFailed,
             driver.instantiate_vnf, self.context, vnf_instance_obj, vnf_dict,
             instantiate_vnf_req_obj)
@@ -414,7 +648,8 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         scale_status = objects.ScaleInfo(aspect_id='SP1', scale_level=0)
         vnf_dict = {"vnfd": {"attributes": {}},
                     "attributes": {"scaling_group_names": {"SP1": "G1"}},
-                    "scale_status": [scale_status]}
+                    "scale_status": [scale_status],
+                    "before_error_point": EP.INITIAL}
         error = self.assertRaises(exceptions.VnfInstantiationWaitFailed,
             driver.instantiate_vnf, self.context, vnf_instance_obj, vnf_dict,
             instantiate_vnf_req_obj)
@@ -468,7 +703,9 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             fake_csar, "sample_vnf_package_csar_with_short_notation")
         self._mock_vnf_manager(vnf_resource_count=2)
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL}
         driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
                                instantiate_vnf_req_obj)
         self.assertEqual(2, mock_create.call_count)
@@ -514,7 +751,9 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             fake_csar, "sample_vnfpkg_no_meta_single_vnfd")
         self._mock_vnf_manager(vnf_resource_count=2)
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {}}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL}
         driver.instantiate_vnf(self.context, vnf_instance_obj, vnf_dict,
                                instantiate_vnf_req_obj)
         self.assertEqual(2, mock_create.call_count)
@@ -553,11 +792,214 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req)
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.INITIAL}
+        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req,
+                             vnf_dict)
         self.assertEqual(2, mock_vnf_instance_save.call_count)
         self.assertEqual(1, mock_resource_destroy.call_count)
         self.assertEqual(5, self._vnf_manager.invoke.call_count)
 
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch.object(vim_client.VimClient, "get_vim")
+    @mock.patch.object(objects.VnfResourceList, "get_by_vnf_instance_id")
+    @mock.patch.object(objects.VnfResource, "destroy")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_terminate_vnf_error_point_vnf_config_start(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_resource_destroy, mock_resource_list,
+            mock_vim, mock_vnf_instance_save, mock_init_hash,
+            mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        vnf_instance.instantiated_vnf_info.instance_id =\
+            uuidsentinel.instance_id
+
+        mock_resource_list.return_value = [fakes.return_vnf_resource()]
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.FORCEFUL)
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.VNF_CONFIG_START}
+        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req,
+                             vnf_dict)
+        self.assertEqual(2, mock_vnf_instance_save.call_count)
+        self.assertEqual(1, mock_resource_destroy.call_count)
+        self.assertEqual(5, self._vnf_manager.invoke.call_count)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch.object(vim_client.VimClient, "get_vim")
+    @mock.patch.object(objects.VnfResourceList, "get_by_vnf_instance_id")
+    @mock.patch.object(objects.VnfResource, "destroy")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_terminate_vnf_error_point_pre_vim_control(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_resource_destroy, mock_resource_list,
+            mock_vim, mock_vnf_instance_save, mock_init_hash,
+            mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        vnf_instance.instantiated_vnf_info.instance_id =\
+            uuidsentinel.instance_id
+
+        mock_resource_list.return_value = [fakes.return_vnf_resource()]
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.FORCEFUL)
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.PRE_VIM_CONTROL}
+        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req,
+                             vnf_dict)
+        self.assertEqual(2, mock_vnf_instance_save.call_count)
+        self.assertEqual(1, mock_resource_destroy.call_count)
+        self.assertEqual(4, self._vnf_manager.invoke.call_count)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch.object(vim_client.VimClient, "get_vim")
+    @mock.patch.object(objects.VnfResourceList, "get_by_vnf_instance_id")
+    @mock.patch.object(objects.VnfResource, "destroy")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_terminate_vnf_error_point_post_vim_control(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_resource_destroy, mock_resource_list,
+            mock_vim, mock_vnf_instance_save, mock_init_hash,
+            mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        vnf_instance.instantiated_vnf_info.instance_id =\
+            uuidsentinel.instance_id
+
+        mock_resource_list.return_value = [fakes.return_vnf_resource()]
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.FORCEFUL)
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.POST_VIM_CONTROL}
+        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req,
+                             vnf_dict)
+        self.assertEqual(2, mock_vnf_instance_save.call_count)
+        self.assertEqual(1, mock_resource_destroy.call_count)
+        self.assertEqual(4, self._vnf_manager.invoke.call_count)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch.object(vim_client.VimClient, "get_vim")
+    @mock.patch.object(objects.VnfResourceList, "get_by_vnf_instance_id")
+    @mock.patch.object(objects.VnfResource, "destroy")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_terminate_vnf_error_point_internal_processing(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_resource_destroy, mock_resource_list,
+            mock_vim, mock_vnf_instance_save, mock_init_hash,
+            mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        vnf_instance.instantiated_vnf_info.instance_id =\
+            uuidsentinel.instance_id
+
+        mock_resource_list.return_value = [fakes.return_vnf_resource()]
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.FORCEFUL)
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.INTERNAL_PROCESSING}
+        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req,
+                             vnf_dict)
+        self.assertEqual(1, mock_vnf_instance_save.call_count)
+        self.assertEqual(0, mock_resource_destroy.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch.object(vim_client.VimClient, "get_vim")
+    @mock.patch.object(objects.VnfResourceList, "get_by_vnf_instance_id")
+    @mock.patch.object(objects.VnfResource, "destroy")
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_terminate_vnf_error_point_vnf_config_end(
+            self, mock_vnf_interfaces, mock_vnfd_dict,
+            mock_resource_destroy, mock_resource_list,
+            mock_vim, mock_vnf_instance_save, mock_init_hash,
+            mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED)
+        vnf_instance.instantiated_vnf_info.instance_id =\
+            uuidsentinel.instance_id
+
+        mock_resource_list.return_value = [fakes.return_vnf_resource()]
+        terminate_vnf_req = objects.TerminateVnfRequest(
+            termination_type=fields.VnfInstanceTerminationType.FORCEFUL)
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.VNF_CONFIG_END}
+        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req,
+                             vnf_dict)
+        self.assertEqual(1, mock_vnf_instance_save.call_count)
+        self.assertEqual(0, mock_resource_destroy.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
+
     @mock.patch.object(TackerManager, 'get_service_plugins',
         return_value={'VNFM': FakeVNFMPlugin()})
     @mock.patch.object(VnfLcmDriver,
@@ -590,7 +1032,10 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req)
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.POST_VIM_CONTROL}
+        driver.terminate_vnf(self.context, vnf_instance, terminate_vnf_req,
+                             vnf_dict)
         self.assertEqual(2, mock_vnf_instance_save.call_count)
         self.assertEqual(1, mock_resource_destroy.call_count)
 
@@ -621,11 +1066,13 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         self._mock_vnf_manager(fail_method_name='delete')
         driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.POST_VIM_CONTROL}
         error = self.assertRaises(InfraDriverException, driver.terminate_vnf,
-            self.context, vnf_instance, terminate_vnf_req)
+            self.context, vnf_instance, terminate_vnf_req, vnf_dict)
         self.assertEqual("delete failed", str(error))
         self.assertEqual(1, mock_vnf_instance_save.call_count)
-        self.assertEqual(2, self._vnf_manager.invoke.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
 
     @mock.patch.object(TackerManager, 'get_service_plugins',
         return_value={'VNFM': FakeVNFMPlugin()})
@@ -654,11 +1101,13 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         self._mock_vnf_manager(fail_method_name='delete_wait')
         driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.POST_VIM_CONTROL}
         error = self.assertRaises(InfraDriverException, driver.terminate_vnf,
-            self.context, vnf_instance, terminate_vnf_req)
+            self.context, vnf_instance, terminate_vnf_req, vnf_dict)
         self.assertEqual("delete_wait failed", str(error))
         self.assertEqual(2, mock_vnf_instance_save.call_count)
-        self.assertEqual(3, self._vnf_manager.invoke.call_count)
+        self.assertEqual(2, self._vnf_manager.invoke.call_count)
 
     @mock.patch.object(TackerManager, 'get_service_plugins',
         return_value={'VNFM': FakeVNFMPlugin()})
@@ -690,11 +1139,13 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         mock_resource_list.return_value = [fakes.return_vnf_resource()]
         self._mock_vnf_manager(fail_method_name='delete_vnf_resource')
         driver = vnflcm_driver.VnfLcmDriver()
+        vnf_dict = {"vnfd": {"attributes": {}}, "attributes": {},
+                    "before_error_point": EP.POST_VIM_CONTROL}
         error = self.assertRaises(InfraDriverException, driver.terminate_vnf,
-            self.context, vnf_instance, terminate_vnf_req)
+            self.context, vnf_instance, terminate_vnf_req, vnf_dict)
         self.assertEqual("delete_vnf_resource failed", str(error))
         self.assertEqual(2, mock_vnf_instance_save.call_count)
-        self.assertEqual(4, self._vnf_manager.invoke.call_count)
+        self.assertEqual(3, self._vnf_manager.invoke.call_count)
 
     @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
     @mock.patch.object(VnfLcmDriver,
@@ -755,7 +1206,8 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         driver = vnflcm_driver.VnfLcmDriver()
         scale_status = objects.ScaleInfo(aspect_id='SP1', scale_level=0)
         vnf_dict = {"attributes": {"scaling_group_names": {"SP1": "G1"}},
-                    "scale_status": [scale_status]}
+                    "scale_status": [scale_status],
+                    "before_error_point": EP.PRE_VIM_CONTROL}
         mock_make_final_vnf_dict.return_value = {}
         driver.heal_vnf(self.context, vnf_instance, vnf_dict, heal_vnf_req)
         self.assertEqual(1, mock_save.call_count)
@@ -767,7 +1219,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         self.assertEqual(1, mock_resource_create.call_count)
         # Invoke will be called 7 times, 3 for deleting the vnf
         # resources  and 4 during instantiation.
-        self.assertEqual(9, self._vnf_manager.invoke.call_count)
+        self.assertEqual(8, self._vnf_manager.invoke.call_count)
         expected_msg = ("Request received for healing vnf '%s' "
                        "is completed successfully")
         mock_log.info.assert_called_with(expected_msg,
@@ -802,18 +1254,17 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             uuidsentinel.instance_id
         self._mock_vnf_manager(fail_method_name='delete')
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"fake": "fake_dict", "grant": None}
+        vnf_dict = {
+            "fake": "fake_dict", "grant": None,
+            "before_error_point": EP.PRE_VIM_CONTROL}
         self.assertRaises(exceptions.VnfHealFailed,
             driver.heal_vnf, self.context, vnf_instance,
             vnf_dict, heal_vnf_req)
         self.assertEqual(1, mock_save.call_count)
-        self.assertEqual(2, self._vnf_manager.invoke.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
         self.assertEqual(fields.VnfInstanceTaskState.ERROR,
             vnf_instance.task_state)
-        expected_msg = ('Failed to delete vnf resources for vnf instance %s '
+        self.assertEqual(mock_log.error.call_count, 2)
-                        'before respawning. The vnf is in inconsistent '
-                        'state. Error: delete failed')
-        mock_log.error.assert_called_with(expected_msg % vnf_instance.id)
 
     @mock.patch('tacker.vnflcm.utils._make_final_vnf_dict')
     @mock.patch.object(VnfLcmDriver,
@@ -861,7 +1312,8 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             uuidsentinel.instance_id
         self._mock_vnf_manager(fail_method_name='instantiate_vnf')
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"fake": "fake_dict"}
+        vnf_dict = {"fake": "fake_dict",
+            "before_error_point": EP.PRE_VIM_CONTROL}
         mock_make_final_vnf_dict.return_value = {}
         self.assertRaises(exceptions.VnfHealFailed,
                           driver.heal_vnf, self.context,
@@ -874,15 +1326,10 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         # instantiation.
         self.assertEqual(1, mock_resource_create.call_count)
 
-        self.assertEqual(6, self._vnf_manager.invoke.call_count)
+        self.assertEqual(5, self._vnf_manager.invoke.call_count)
         self.assertEqual(fields.VnfInstanceTaskState.ERROR,
             vnf_instance.task_state)
-        expected_msg = ('Failed to instantiate vnf instance %s '
+        self.assertEqual(mock_log.error.call_count, 3)
-                        'after termination. The vnf is in inconsistent '
-                        'state. Error: Vnf instantiation failed for vnf %s, '
-                        'error: instantiate_vnf failed')
-        mock_log.error.assert_called_with(expected_msg % (vnf_instance.id,
-            vnf_instance.id))
         mock_final_vnf_dict.assert_called_once()
 
     @mock.patch.object(TackerManager, 'get_service_plugins',
@@ -909,9 +1356,12 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             fields.VnfInstanceState.INSTANTIATED,
             task_state=fields.VnfInstanceTaskState.HEALING)
 
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL}
+
         self._mock_vnf_manager()
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"grant": None}
         driver.heal_vnf(self.context, vnf_instance, vnf_dict, heal_vnf_req)
         self.assertEqual(1, mock_save.call_count)
         self.assertEqual(5, self._vnf_manager.invoke.call_count)
@@ -922,6 +1372,201 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         mock_log.info.assert_called_with(expected_msg,
             vnf_instance.id)
 
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.vnflcm_driver.LOG')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_heal_vnf_with_vnfc_instance_error_point_vnf_config_start(
+            self, mock_vnf_interfaces, mock_vnfd_dict, mock_log, mock_save,
+            mock_init_hash, mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        heal_vnf_req = objects.HealVnfRequest(vnfc_instance_id=[
+            uuidsentinel.vnfc_instance_id_1])
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED,
+            task_state=fields.VnfInstanceTaskState.HEALING)
+
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.VNF_CONFIG_START,
+            "grant": None}
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        driver.heal_vnf(self.context, vnf_instance, vnf_dict, heal_vnf_req)
+        self.assertEqual(1, mock_save.call_count)
+        self.assertEqual(5, self._vnf_manager.invoke.call_count)
+
+        self.assertEqual(None, vnf_instance.task_state)
+        expected_msg = ("Request received for healing vnf '%s' "
+                       "is completed successfully")
+        mock_log.info.assert_called_with(expected_msg,
+            vnf_instance.id)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.vnflcm_driver.LOG')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_heal_vnf_with_vnfc_instance_error_point_pre_vim_control(
+            self, mock_vnf_interfaces, mock_vnfd_dict, mock_log, mock_save,
+            mock_init_hash, mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        heal_vnf_req = objects.HealVnfRequest(vnfc_instance_id=[
+            uuidsentinel.vnfc_instance_id_1])
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED,
+            task_state=fields.VnfInstanceTaskState.HEALING)
+
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.PRE_VIM_CONTROL}
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        driver.heal_vnf(self.context, vnf_instance, vnf_dict, heal_vnf_req)
+        self.assertEqual(1, mock_save.call_count)
+        self.assertEqual(4, self._vnf_manager.invoke.call_count)
+
+        self.assertEqual(None, vnf_instance.task_state)
+        expected_msg = ("Request received for healing vnf '%s' "
+                       "is completed successfully")
+        mock_log.info.assert_called_with(expected_msg,
+            vnf_instance.id)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.vnflcm_driver.LOG')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_heal_vnf_with_vnfc_instance_error_point_post_vim_control(
+            self, mock_vnf_interfaces, mock_vnfd_dict, mock_log, mock_save,
+            mock_init_hash, mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        heal_vnf_req = objects.HealVnfRequest(vnfc_instance_id=[
+            uuidsentinel.vnfc_instance_id_1])
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED,
+            task_state=fields.VnfInstanceTaskState.HEALING)
+
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.POST_VIM_CONTROL}
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        driver.heal_vnf(self.context, vnf_instance, vnf_dict, heal_vnf_req)
+        self.assertEqual(1, mock_save.call_count)
+        self.assertEqual(4, self._vnf_manager.invoke.call_count)
+
+        self.assertEqual(None, vnf_instance.task_state)
+        expected_msg = ("Request received for healing vnf '%s' "
+                       "is completed successfully")
+        mock_log.info.assert_called_with(expected_msg,
+            vnf_instance.id)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.vnflcm_driver.LOG')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_heal_vnf_with_vnfc_instance_error_point_internal_processing(
+            self, mock_vnf_interfaces, mock_vnfd_dict, mock_log, mock_save,
+            mock_init_hash, mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        heal_vnf_req = objects.HealVnfRequest(vnfc_instance_id=[
+            uuidsentinel.vnfc_instance_id_1])
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED,
+            task_state=None)
+
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INTERNAL_PROCESSING}
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        driver.heal_vnf(self.context, vnf_instance, vnf_dict, heal_vnf_req)
+        self.assertEqual(0, mock_save.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
+
+        self.assertEqual(None, vnf_instance.task_state)
+        self.assertEqual(1, mock_log.info.call_count)
+
+    @mock.patch.object(TackerManager, 'get_service_plugins',
+        return_value={'VNFM': FakeVNFMPlugin()})
+    @mock.patch.object(VnfLcmDriver,
+                       '_init_mgmt_driver_hash')
+    @mock.patch.object(objects.VnfInstance, "save")
+    @mock.patch('tacker.vnflcm.vnflcm_driver.LOG')
+    @mock.patch('tacker.vnflcm.utils._get_vnfd_dict')
+    @mock.patch('tacker.vnflcm.vnflcm_driver.VnfLcmDriver.'
+                '_load_vnf_interface')
+    def test_heal_vnf_with_vnfc_instance_error_point_vnf_config_end(
+            self, mock_vnf_interfaces, mock_vnfd_dict, mock_log, mock_save,
+            mock_init_hash, mock_get_service_plugins):
+        mock_init_hash.return_value = {
+            "vnflcm_noop": "ffea638bfdbde3fb01f191bbe75b031859"
+                           "b18d663b127100eb72b19eecd7ed51"
+        }
+        mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+        heal_vnf_req = objects.HealVnfRequest(vnfc_instance_id=[
+            uuidsentinel.vnfc_instance_id_1])
+
+        vnf_instance = fakes.return_vnf_instance(
+            fields.VnfInstanceState.INSTANTIATED,
+            task_state=None)
+
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.VNF_CONFIG_END}
+
+        self._mock_vnf_manager()
+        driver = vnflcm_driver.VnfLcmDriver()
+        driver.heal_vnf(self.context, vnf_instance, vnf_dict, heal_vnf_req)
+        self.assertEqual(0, mock_save.call_count)
+        self.assertEqual(1, self._vnf_manager.invoke.call_count)
+
+        self.assertEqual(None, vnf_instance.task_state)
+        self.assertEqual(1, mock_log.info.call_count)
+
     @mock.patch.object(TackerManager, 'get_service_plugins',
         return_value={'VNFM': FakeVNFMPlugin()})
     @mock.patch.object(VnfLcmDriver,
@@ -945,9 +1590,15 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             fields.VnfInstanceState.INSTANTIATED,
             task_state=fields.VnfInstanceTaskState.HEALING)
         mock_vnf_interfaces.return_value = fakes.return_vnf_interfaces()
+
+        vnf_dict = mock.ANY
+
         self._mock_vnf_manager(fail_method_name='heal_vnf')
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"grant": None}
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL,
+            "grant": None}
         self.assertRaises(exceptions.VnfHealFailed,
             driver.heal_vnf, self.context, vnf_instance,
             vnf_dict, heal_vnf_req)
@@ -956,10 +1607,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         self.assertEqual(fields.VnfInstanceTaskState.ERROR,
             vnf_instance.task_state)
-        expected_msg = ("Failed to heal vnf %(id)s in infra driver. "
+        self.assertEqual(mock_log.error.call_count, 2)
-                       "Error: %(error)s")
-        mock_log.error.assert_called_with(expected_msg,
-            {'id': vnf_instance.id, 'error': 'heal_vnf failed'})
 
     @mock.patch.object(TackerManager, 'get_service_plugins',
         return_value={'VNFM': FakeVNFMPlugin()})
@@ -987,10 +1635,13 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         vnf_instance.instantiated_vnf_info.instance_id =\
             uuidsentinel.instance_id
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL,
+            "grant": None}
         self._mock_vnf_manager(fail_method_name='heal_vnf_wait')
         driver = vnflcm_driver.VnfLcmDriver()
 
-        vnf_dict = {"grant": None}
         # It won't raise any exception if infra driver raises
         # heal_vnf_wait because there is a possibility the vnfc
         # resources could go into inconsistent state so it would
@@ -1037,9 +1688,12 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         vnf_instance.instantiated_vnf_info.instance_id =\
             uuidsentinel.instance_id
+        vnf_dict = {
+            "vnfd": {"attributes": {}}, "attributes": {},
+            "before_error_point": EP.INITIAL,
+            "grant": None}
         self._mock_vnf_manager(fail_method_name='post_heal_vnf')
         driver = vnflcm_driver.VnfLcmDriver()
-        vnf_dict = {"grant": None}
         self.assertRaises(exceptions.VnfHealFailed,
             driver.heal_vnf, self.context, vnf_instance,
             vnf_dict, heal_vnf_req)
@@ -1048,13 +1702,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
 
         self.assertEqual(fields.VnfInstanceTaskState.ERROR,
             vnf_instance.task_state)
-        expected_msg = ('Failed to store updated resources information for '
+        self.assertEqual(mock_log.error.call_count, 2)
-                        'instance %(instance)s for vnf %(id)s. '
-                        'Error: %(error)s')
-        mock_log.error.assert_called_with(expected_msg,
-            {'instance': vnf_instance.instantiated_vnf_info.instance_id,
-             'id': vnf_instance.id,
-             'error': 'post_heal_vnf failed'})
 
     @mock.patch.object(TackerManager, 'get_service_plugins',
         return_value={'VNFM': FakeVNFMPlugin()})
@@ -1073,6 +1721,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             '1, \"maxLevel\": 3, \"initialNum\": 0, ' + \
             '\"initialLevel\": 0, \"default\": 0 }}}'
         scale_vnf_request = fakes.scale_request("SCALE_IN", "SP1", 1, "True")
+        vnf_info['vnf_lcm_op_occ'] = mock.ANY
         vim_connection_info = vim_connection.VimConnectionInfo(
             vim_type="openstack")
         scale_name_list = ["fake"]
@@ -1100,6 +1749,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             '1, \"maxLevel\": 3, \"initialNum\": 0, ' + \
             '\"initialLevel\": 0, \"default\": 0 }}}'
         scale_vnf_request = fakes.scale_request("SCALE_IN", "SP1", 1, "False")
+        vnf_info['vnf_lcm_op_occ'] = mock.ANY
         vim_connection_info = vim_connection.VimConnectionInfo(
             vim_type="openstack")
         scale_name_list = ["fake"]
@@ -1130,6 +1780,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             '1, \"maxLevel\": 3, \"initialNum\": 0, ' + \
             '\"initialLevel\": 0, \"default\": 0 }}}'
         scale_vnf_request = fakes.scale_request("SCALE_OUT", "SP1", 1, "False")
+        vnf_info['vnf_lcm_op_occ'] = mock.ANY
         vim_connection_info = vim_connection.VimConnectionInfo(
             vim_type="openstack")
         scale_name_list = ["fake"]
@@ -1160,6 +1811,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
             '1, \"maxLevel\": 3, \"initialNum\": 0, ' + \
             '\"initialLevel\": 0, \"default\": 1 }}}'
         scale_vnf_request = fakes.scale_request("SCALE_OUT", "SP1", 1, "False")
+        vnf_info['vnf_lcm_op_occ'] = mock.ANY
         vim_connection_info = vim_connection.VimConnectionInfo(
             vim_type="openstack")
         scale_name_list = ["fake"]
@@ -1195,6 +1847,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         vnf_info['scale_level'] = 1
         vnf_info['after_scale_level'] = 0
         vnf_info['notification'] = {}
+        vnf_info['before_error_point'] = EP.INITIAL
         scale_vnf_request = fakes.scale_request(
             "SCALE_IN", "vdu1_aspect", 1, "False")
         vim_connection_info = vim_connection.VimConnectionInfo(
@@ -1246,6 +1899,7 @@ class TestVnflcmDriver(db_base.SqlTestCase):
         vnf_info['scale_level'] = 0
         vnf_info['after_scale_level'] = 1
         vnf_info['notification'] = {}
+        vnf_info['before_error_point'] = EP.INITIAL
         scale_vnf_request = fakes.scale_request(
             "SCALE_OUT", "vdu1_aspect", 1, "False")
         vim_connection_info = vim_connection.VimConnectionInfo(

tacker/tests/unit/vnfm/infra_drivers/openstack/fixture_data/fixture_data_utils.py

@@ -330,3 +330,19 @@ def get_grant_response_dict():
             resource_name='VirtualStorage')]}
 
     return grant_response_dict
+
+
+def get_lcm_op_occs_object(operation="INSTANTIATE",
+        error_point=0):
+    vnf_lcm_op_occs = objects.VnfLcmOpOcc(
+        id=uuidsentinel.lcm_op_occs_id,
+        tenant_id=uuidsentinel.tenant_id,
+        operation_state='PROCESSING',
+        state_entered_time='2019-03-06T05:44:27Z',
+        start_time='2019-03-06T05:44:27Z',
+        operation=operation,
+        is_automatic_invocation=0,
+        is_cancel_pending=0,
+        error_point=error_point)
+
+    return vnf_lcm_op_occs

tacker/tests/unit/vnfm/infra_drivers/openstack/test_openstack.py

@@ -23,6 +23,7 @@ from oslo_serialization import jsonutils
 from tacker import context
 from tacker.db.common_services import common_services_db_plugin
 from tacker.extensions import vnfm
+from tacker.objects import fields
 from tacker.tests.unit import base
 from tacker.tests.unit.db import utils
 from tacker.vnfm.infra_drivers.openstack import openstack
@@ -377,9 +378,11 @@ class TestOpenStack(base.TestCase):
                                                     input_params,
                                                     is_monitor,
                                                     multi_vdus)
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         result = self.infra_driver.create(plugin=None, context=self.context,
                                          vnf=vnf,
                                          auth_attr=utils.get_vim_auth_obj())
+        del vnf['before_error_point']
         actual_fields = self.heat_client.create.call_args[0][0]
         actual_fields["template"] = yaml.safe_load(actual_fields["template"])
         expected_fields["template"] = \

tacker/tests/unit/vnfm/infra_drivers/openstack/test_openstack_driver.py

@@ -31,6 +31,7 @@ from tacker import context
 from tacker.db.db_sqlalchemy import models
 from tacker.extensions import vnfm
 from tacker import objects
+from tacker.objects import fields
 from tacker.tests import constants
 from tacker.tests.unit import base
 from tacker.tests.unit.db import utils
@@ -183,12 +184,54 @@ class TestOpenStack(base.FixturedTestCase):
         mock_get_base_hot_dict.return_value = \
             self._read_file(), nested_hot_dict
         vnf_instance = fd_utils.get_vnf_instance_object()
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         self.openstack.create(self.plugin, self.context, vnf,
                 self.auth_attr, inst_req_info=inst_req_info_test,
                 vnf_package_path=vnf_package_path_test,
                 grant_info=grant_info_test,
                 vnf_instance=vnf_instance)
 
+    @mock.patch('tacker.vnfm.infra_drivers.openstack.openstack'
+                '.OpenStack._format_base_hot')
+    @mock.patch('tacker.vnflcm.utils._get_vnflcm_interface')
+    @mock.patch('tacker.vnflcm.utils.get_base_nest_hot_dict')
+    @mock.patch('tacker.common.clients.OpenstackClients')
+    @mock.patch('tacker.vnfm.infra_drivers.openstack.openstack'
+                '.OpenStack._update_stack_with_user_data')
+    @mock.patch.object(hc.HeatClient, "find_stack")
+    def test_create_normal_with_error_point_post_vim_control(
+            self, mock_find_stack, mock_update_stack_with_user_data,
+            mock_OpenstackClients_heat, mock_get_base_hot_dict,
+            mock_get_vnflcm_interface, mock_format_base_hot):
+        vnf = utils.get_dummy_vnf_etsi(instance_id=self.instance_uuid,
+                                       flavour='simple')
+        vnf['placement_attr'] = {'region_name': 'dummy_region'}
+        vnf_package_path_test = os.path.abspath(
+            os.path.join(os.path.dirname(__file__),
+                         "../../../../etc/samples/etsi/nfv",
+                         "user_data_sample_normal"))
+        inst_req_info_test = type('', (), {})
+        test_json = self._json_load(
+            'instantiate_vnf_request_lcm_userdata.json')
+        inst_req_info_test.additional_params = test_json['additionalParams']
+        inst_req_info_test.ext_virtual_links = None
+        inst_req_info_test.flavour_id = 'simple'
+        vnf_resource = type('', (), {})
+        vnf_resource.resource_identifier = constants.INVALID_UUID
+        grant_info_test = {'vdu_name': {vnf_resource}}
+        nested_hot_dict = {'parameters': {'vnf': 'test'}}
+        mock_get_base_hot_dict.return_value = \
+            self._read_file(), nested_hot_dict
+        vnf_instance = fd_utils.get_vnf_instance_object()
+        vnf['before_error_point'] = fields.ErrorPoint.POST_VIM_CONTROL
+        self.openstack.create(self.plugin, self.context, vnf,
+                self.auth_attr, inst_req_info=inst_req_info_test,
+                vnf_package_path=vnf_package_path_test,
+                grant_info=grant_info_test,
+                vnf_instance=vnf_instance)
+        mock_find_stack.assert_called_once()
+        mock_update_stack_with_user_data.assert_called_once()
+
     @mock.patch('tacker.vnfm.vim_client.VimClient.get_vim')
     @mock.patch('tacker.vnfm.infra_drivers.openstack.openstack'
                 '.OpenStack._format_base_hot')
@@ -289,6 +332,7 @@ class TestOpenStack(base.FixturedTestCase):
             resource_id='6e1c286d-c023-4b34-8369-831c6e84cce2')
         vnfc_obj.compute_resource = compute_resource
         vnf_instance.instantiated_vnf_info.vnfc_resource_info = [vnfc_obj]
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         self.openstack.create(self.plugin, self.context, vnf,
                 self.auth_attr, inst_req_info=inst_req_info_test,
                 vnf_package_path=vnf_package_path_test,
@@ -300,9 +344,26 @@ class TestOpenStack(base.FixturedTestCase):
         vnf = utils.get_dummy_vnf(instance_id=self.instance_uuid)
         vnf['placement_attr'] = {'region_name': 'dummy_region'}
         vnf_package_path = None
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         self.openstack.create(self.plugin, self.context, vnf,
                 self.auth_attr, vnf_package_path)
 
+    @mock.patch('tacker.common.clients.OpenstackClients')
+    @mock.patch('tacker.vnfm.infra_drivers.openstack.openstack'
+                '.OpenStack._update_stack')
+    @mock.patch.object(hc.HeatClient, "find_stack")
+    def test_create_heat_stack_with_error_point_post_vim_control(self,
+            mock_find_stack, mock_update_stack,
+            mock_OpenstackClients_heat):
+        vnf = utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        vnf['placement_attr'] = {'region_name': 'dummy_region'}
+        vnf_package_path = None
+        vnf['before_error_point'] = fields.ErrorPoint.POST_VIM_CONTROL
+        self.openstack.create(self.plugin, self.context, vnf,
+                self.auth_attr, vnf_package_path)
+        mock_find_stack.assert_called_once()
+        mock_update_stack.assert_called_once()
+
     @mock.patch('tacker.common.clients.OpenstackClients')
     def test_create_userdata_none(self, mock_OpenstackClients_heat):
         vnf = utils.get_dummy_vnf(instance_id=self.instance_uuid)
@@ -511,10 +572,12 @@ class TestOpenStack(base.FixturedTestCase):
         mock_get_base_hot_dict.return_value = \
             self._read_file(), nested_hot_dict
         vnf_instance = fd_utils.get_vnf_instance_object()
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         self.assertRaises(vnfm.LCMUserDataFailed,
                           self.openstack.create,
                           self.plugin, self.context, vnf,
-                          self.auth_attr, inst_req_info=inst_req_info_test,
+                          self.auth_attr,
+                          inst_req_info=inst_req_info_test,
                           vnf_package_path=vnf_package_path_test,
                           grant_info=grant_info_test,
                           vnf_instance=vnf_instance)
@@ -545,10 +608,12 @@ class TestOpenStack(base.FixturedTestCase):
         mock_get_base_hot_dict.return_value = \
             self._read_file(), nested_hot_dict
         vnf_instance = fd_utils.get_vnf_instance_object()
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         self.assertRaises(vnfm.LCMUserDataFailed,
                           self.openstack.create,
                           self.plugin, self.context, vnf,
-                          self.auth_attr, inst_req_info=inst_req_info_test,
+                          self.auth_attr,
+                          inst_req_info=inst_req_info_test,
                           vnf_package_path=vnf_package_path_test,
                           grant_info=grant_info_test,
                           vnf_instance=vnf_instance)
@@ -680,10 +745,12 @@ class TestOpenStack(base.FixturedTestCase):
         mock_get_base_hot_dict.return_value = \
             self._read_file(), nested_hot_dict
         vnf_instance = fd_utils.get_vnf_instance_object()
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         self.assertRaises(vnfm.LCMUserDataFailed,
                           self.openstack.create,
                           self.plugin, self.context, vnf,
-                          self.auth_attr, inst_req_info=inst_req_info_test,
+                          self.auth_attr,
+                          inst_req_info=inst_req_info_test,
                           vnf_package_path=vnf_package_path_test,
                           grant_info=grant_info_test,
                           vnf_instance=vnf_instance)
@@ -758,10 +825,12 @@ class TestOpenStack(base.FixturedTestCase):
         mock_get_base_hot_dict.return_value = \
             self._read_file(), nested_hot_dict
         vnf_instance = fd_utils.get_vnf_instance_object()
+        vnf['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         self.assertRaises(vnfm.LCMUserDataFailed,
                           self.openstack.create,
                           self.plugin, self.context, vnf,
-                          self.auth_attr, inst_req_info=inst_req_info_test,
+                          self.auth_attr,
+                          inst_req_info=inst_req_info_test,
                           vnf_package_path=vnf_package_path_test,
                           grant_info=grant_info_test,
                           vnf_instance=vnf_instance)
@@ -777,6 +846,16 @@ class TestOpenStack(base.FixturedTestCase):
         self.assertEqual(bytes('{"VDU1": "192.168.120.216"}', 'utf-8'),
                          vnf_dict['mgmt_ip_address'])
 
+    @mock.patch('tacker.vnfm.infra_drivers.openstack.openstack'
+                '.OpenStack._wait_until_stack_ready')
+    def test_update_stack_wait(self, mock_wait_until_stack_ready):
+        self._response_in_wait_until_stack_ready(["UPDATE_IN_PROGRESS",
+                                                 "UPDATE_COMPLETE"])
+        vnf_dict = utils.get_dummy_vnf(instance_id=self.instance_uuid)
+        self.openstack.update_stack_wait(None, None,
+                                         vnf_dict, self.instance_uuid, None)
+        mock_wait_until_stack_ready.assert_called_once()
+
     def test_create_wait_without_mgmt_ips(self):
         self._response_in_wait_until_stack_ready(["CREATE_IN_PROGRESS",
                                                  "CREATE_COMPLETE"],
@@ -1327,6 +1406,7 @@ class TestOpenStack(base.FixturedTestCase):
             headers=self.json_headers)
         vnf_instance = fd_utils.get_vnf_instance_object()
 
+        vnfd_dict['before_error_point'] = fields.ErrorPoint.PRE_VIM_CONTROL
         instance_id = self.openstack.instantiate_vnf(
             self.context, vnf_instance, vnfd_dict, vim_connection_info,
             inst_req_info, grant_response, self.plugin)
@@ -1483,7 +1563,8 @@ class TestOpenStack(base.FixturedTestCase):
             ext_managed_virtual_link_info[0].vnf_link_ports[0].
             resource_handle.resource_id)
 
-    def test_heal_vnf_instance(self):
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_vnf_instance_id")
+    def test_heal_vnf_instance(self, mock_get_vnflcm_op_occs):
         v_s_resource_info = fd_utils.get_virtual_storage_resource_info(
             desc_id="storage1")
 
@@ -1527,6 +1608,9 @@ class TestOpenStack(base.FixturedTestCase):
         self._response_resource_mark_unhealthy(inst_vnf_info.instance_id,
                 resources=resources)
 
+        vnf_lcm_op_occs = fd_utils.get_lcm_op_occs_object(
+            error_point=fields.ErrorPoint.PRE_VIM_CONTROL)
+        mock_get_vnflcm_op_occs.return_value = vnf_lcm_op_occs
         self.openstack.heal_vnf(
             self.context, vnf_instance, vim_connection_info, heal_vnf_request)
 
@@ -1536,7 +1620,64 @@ class TestOpenStack(base.FixturedTestCase):
         # as unhealthy, and 1 for updating stack
         self.assertEqual(3, len(patch_req))
 
-    def test_heal_vnf_instance_resource_mark_unhealthy_error(self):
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_vnf_instance_id")
+    def test_heal_vnf_instance_error_point_post_vim_control(
+            self, mock_get_vnflcm_op_occs):
+        v_s_resource_info = fd_utils.get_virtual_storage_resource_info(
+            desc_id="storage1")
+
+        storage_resource_ids = [v_s_resource_info.id]
+        vnfc_resource_info = fd_utils.get_vnfc_resource_info(vdu_id="VDU_VNF",
+            storage_resource_ids=storage_resource_ids)
+
+        inst_vnf_info = fd_utils.get_vnf_instantiated_info(
+            virtual_storage_resource_info=[v_s_resource_info],
+            vnfc_resource_info=[vnfc_resource_info])
+
+        vnf_instance = fd_utils.get_vnf_instance_object(
+            instantiated_vnf_info=inst_vnf_info)
+
+        vim_connection_info = fd_utils.get_vim_connection_info_object()
+
+        heal_vnf_request = objects.HealVnfRequest(
+            vnfc_instance_id=[vnfc_resource_info.id],
+            cause="healing request")
+
+        # Mock various heat APIs that will be called by heatclient
+        # during the process of heal_vnf.
+        resources = [{
+            'resource_name': vnfc_resource_info.vdu_id,
+            'resource_type': vnfc_resource_info.compute_resource.
+            vim_level_resource_type,
+            'physical_resource_id': vnfc_resource_info.compute_resource.
+            resource_id}, {
+            'resource_name': v_s_resource_info.virtual_storage_desc_id,
+            'resource_type': v_s_resource_info.storage_resource.
+            vim_level_resource_type,
+            'physical_resource_id': v_s_resource_info.storage_resource.
+            resource_id}]
+
+        self._response_in_stack_get(inst_vnf_info.instance_id)
+        self._response_in_resource_get_list(inst_vnf_info.instance_id,
+                resources=resources)
+        self._responses_in_stack_list(inst_vnf_info.instance_id,
+            resources=resources)
+        self._response_in_stack_update(inst_vnf_info.instance_id)
+
+        vnf_lcm_op_occs = fd_utils.get_lcm_op_occs_object(
+            error_point=fields.ErrorPoint.POST_VIM_CONTROL)
+        mock_get_vnflcm_op_occs.return_value = vnf_lcm_op_occs
+        self.openstack.heal_vnf(
+            self.context, vnf_instance, vim_connection_info, heal_vnf_request)
+
+        history = self.requests_mock.request_history
+        patch_req = [req.url for req in history if req.method == 'PATCH']
+        # Total of 1 times for updating stack
+        self.assertEqual(1, len(patch_req))
+
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_vnf_instance_id")
+    def test_heal_vnf_instance_resource_mark_unhealthy_error(
+            self, mock_get_vnflcm_op_occs):
         vnfc_resource_info = fd_utils.get_vnfc_resource_info(vdu_id="VDU_VNF")
 
         inst_vnf_info = fd_utils.get_vnf_instantiated_info(
@@ -1568,6 +1709,9 @@ class TestOpenStack(base.FixturedTestCase):
         self._response_resource_mark_unhealthy(inst_vnf_info.instance_id,
                 resources=resources, raise_exception=True)
 
+        vnf_lcm_op_occs = fd_utils.get_lcm_op_occs_object(
+            error_point=3)
+        mock_get_vnflcm_op_occs.return_value = vnf_lcm_op_occs
         result = self.assertRaises(exceptions.VnfHealFailed,
                     self.openstack.heal_vnf, self.context, vnf_instance,
                     vim_connection_info, heal_vnf_request)
@@ -1584,7 +1728,9 @@ class TestOpenStack(base.FixturedTestCase):
         # as unhealthy
         self.assertEqual(1, len(patch_req))
 
-    def test_heal_vnf_instance_incorrect_stack_status(self):
+    @mock.patch.object(objects.VnfLcmOpOcc, "get_by_vnf_instance_id")
+    def test_heal_vnf_instance_incorrect_stack_status(
+            self, mock_get_vnflcm_op_occs):
         inst_vnf_info = fd_utils.get_vnf_instantiated_info()
 
         vnf_instance = fd_utils.get_vnf_instance_object(
@@ -1601,6 +1747,9 @@ class TestOpenStack(base.FixturedTestCase):
         self._response_in_stack_get(inst_vnf_info.instance_id,
                 stack_status='UPDATE_IN_PROGRESS')
 
+        vnf_lcm_op_occs = fd_utils.get_lcm_op_occs_object(
+            error_point=3)
+        mock_get_vnflcm_op_occs.return_value = vnf_lcm_op_occs
         result = self.assertRaises(exceptions.VnfHealFailed,
             self.openstack.heal_vnf, self.context, vnf_instance,
             vim_connection_info, heal_vnf_request)

tacker/vnflcm/vnflcm_driver.py

@@ -40,6 +40,7 @@ from tacker.conductor.conductorrpc import vnf_lcm_rpc
 from tacker import manager
 from tacker import objects
 from tacker.objects import fields
+from tacker.objects.fields import ErrorPoint as EP
 from tacker.vnflcm import abstract_driver
 from tacker.vnflcm import utils as vnflcm_utils
 
@@ -109,6 +110,8 @@ def revert_to_error_scale(function):
                     vnf_lcm_op_occ.state_entered_time = timestamp
                     vnf_lcm_op_occ.resource_changes = resource_changes
                     vnf_lcm_op_occ.error = problem
+                    vnf_lcm_op_occ.error_point = \
+                        vnf_info['current_error_point']
                     vnf_lcm_op_occ.save()
                 except Exception as e:
                     LOG.warning("Failed to update vnf_lcm_op_occ for vnf "
@@ -361,6 +364,9 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             vnfd_dict_to_create_final_dict, vnf_instance.id,
             vnf_instance.vnf_instance_name, param_for_subs_map, vnf_dict)
 
+        final_vnf_dict['before_error_point'] = \
+            vnf_dict['before_error_point']
+
         try:
             instance_id = self._vnf_manager.invoke(
                 vim_connection_info.vim_type, 'instantiate_vnf',
@@ -398,24 +404,44 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                 vnf_instance.instantiated_vnf_info.scale_status = \
                     default_scale_status
 
-        try:
+        if vnf_dict['before_error_point'] <= EP.PRE_VIM_CONTROL:
-            self._vnf_manager.invoke(
+            try:
-                vim_connection_info.vim_type, 'create_wait',
+                self._vnf_manager.invoke(
-                plugin=self._vnfm_plugin, context=context,
+                    vim_connection_info.vim_type, 'create_wait',
-                vnf_dict=final_vnf_dict,
+                    plugin=self._vnfm_plugin, context=context,
-                vnf_id=final_vnf_dict['instance_id'],
+                    vnf_dict=final_vnf_dict,
-                auth_attr=vim_connection_info.access_info)
+                    vnf_id=final_vnf_dict['instance_id'],
+                    auth_attr=vim_connection_info.access_info)
 
-        except Exception as exp:
+            except Exception as exp:
-            with excutils.save_and_reraise_exception():
+                with excutils.save_and_reraise_exception():
-                exp.reraise = False
+                    exp.reraise = False
-                LOG.error("Vnf creation wait failed for vnf instance "
+                    LOG.error("Vnf creation wait failed for vnf instance "
-                    "%(id)s due to error : %(error)s",
+                        "%(id)s due to error : %(error)s",
-                    {"id": vnf_instance.id, "error":
+                        {"id": vnf_instance.id, "error":
-                    encodeutils.exception_to_unicode(exp)})
+                        encodeutils.exception_to_unicode(exp)})
-                raise exceptions.VnfInstantiationWaitFailed(
+                    raise exceptions.VnfInstantiationWaitFailed(
-                    id=vnf_instance.id,
+                        id=vnf_instance.id,
-                    error=encodeutils.exception_to_unicode(exp))
+                        error=encodeutils.exception_to_unicode(exp))
+        elif vnf_dict['before_error_point'] == EP.POST_VIM_CONTROL:
+            try:
+                self._vnf_manager.invoke(
+                    vim_connection_info.vim_type, 'update_stack_wait',
+                    plugin=self._vnfm_plugin, context=context,
+                    vnf_dict=final_vnf_dict,
+                    stack_id=instance_id,
+                    auth_attr=vim_connection_info.access_info)
+
+            except Exception as exp:
+                with excutils.save_and_reraise_exception():
+                    exp.reraise = False
+                    LOG.error("Vnf update wait failed for vnf instance "
+                        "%(id)s due to error : %(error)s",
+                        {"id": vnf_instance.id, "error":
+                        encodeutils.exception_to_unicode(exp)})
+                    raise exceptions.VnfInstantiationWaitFailed(
+                        id=vnf_instance.id,
+                        error=encodeutils.exception_to_unicode(exp))
 
     def _get_file_hash(self, path):
         hash_obj = hashlib.sha256()
@@ -472,6 +498,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
     def instantiate_vnf(self, context, vnf_instance, vnf_dict,
             instantiate_vnf_req):
 
+        vnf_dict['current_error_point'] = EP.VNF_CONFIG_START
         vim_connection_info_list = vnflcm_utils.\
             _get_vim_connection_info_from_vnf_req(vnf_instance,
                     instantiate_vnf_req)
@@ -488,38 +515,46 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
         vnfd_dict = vnflcm_utils._get_vnfd_dict(
             context, vnf_instance.vnfd_id, instantiate_vnf_req.flavour_id)
 
-        # TODO(LiangLu): grant_request here is planned to pass
+        if vnf_dict['before_error_point'] <= EP.VNF_CONFIG_START:
-        # as a parameter, however due to grant_request is not
+            # TODO(LiangLu): grant_request here is planned to pass
-        # passed from conductor to vnflcm_driver, thus we put Null
+            # as a parameter, however due to grant_request is not
-        # value to grant_reqeust temporary.
+            # passed from conductor to vnflcm_driver, thus we put Null
-        # This part will be updated in next release.
+            # value to grant_reqeust temporary.
-        self._mgmt_manager.invoke(
+            # This part will be updated in next release.
-            self._load_vnf_interface(
+            self._mgmt_manager.invoke(
-                context, 'instantiate_start', vnf_instance, vnfd_dict),
+                self._load_vnf_interface(
-            'instantiate_start', context=context,
+                    context, 'instantiate_start', vnf_instance, vnfd_dict),
-            vnf_instance=vnf_instance,
+                'instantiate_start', context=context,
-            instantiate_vnf_request=instantiate_vnf_req,
+                vnf_instance=vnf_instance,
-            grant=vnf_dict.get('grant'), grant_request=None)
+                instantiate_vnf_request=instantiate_vnf_req,
+                grant=vnf_dict.get('grant'), grant_request=None)
 
-        self._instantiate_vnf(context, vnf_instance, vnf_dict,
+        vnf_dict['current_error_point'] = EP.PRE_VIM_CONTROL
-                              vim_connection_info, instantiate_vnf_req)
+        if vnf_dict['before_error_point'] <= EP.POST_VIM_CONTROL:
+            self._instantiate_vnf(context, vnf_instance, vnf_dict,
+                                vim_connection_info, instantiate_vnf_req)
 
-        # TODO(LiangLu): grant_request here is planned to pass
+        vnf_dict['current_error_point'] = EP.INTERNAL_PROCESSING
-        # as a parameter, however due to grant_request is not
+        vnf_dict['current_error_point'] = EP.VNF_CONFIG_END
-        # passed from conductor to vnflcm_driver, thus we put Null
+        if vnf_dict['before_error_point'] <= EP.VNF_CONFIG_END:
-        # value to grant_reqeust temporary.
+            # TODO(LiangLu): grant_request here is planned to pass
-        # This part will be updated in next release.
+            # as a parameter, however due to grant_request is not
-        self._mgmt_manager.invoke(
+            # passed from conductor to vnflcm_driver, thus we put Null
-            self._load_vnf_interface(
+            # value to grant_reqeust temporary.
-                context, 'instantiate_end', vnf_instance, vnfd_dict),
+            # This part will be updated in next release.
-            'instantiate_end', context=context,
+            self._mgmt_manager.invoke(
-            vnf_instance=vnf_instance,
+                self._load_vnf_interface(
-            instantiate_vnf_request=instantiate_vnf_req,
+                    context, 'instantiate_end', vnf_instance, vnfd_dict),
-            grant=vnf_dict.get('grant'), grant_request=None)
+                'instantiate_end', context=context,
+                vnf_instance=vnf_instance,
+                instantiate_vnf_request=instantiate_vnf_req,
+                grant=vnf_dict.get('grant'), grant_request=None)
 
     @log.log
     @revert_to_error_task_state
-    def terminate_vnf(self, context, vnf_instance, terminate_vnf_req):
+    def terminate_vnf(self, context, vnf_instance, terminate_vnf_req,
+            vnf_dict):
+        vnf_dict['current_error_point'] = EP.VNF_CONFIG_START
 
         vim_info = vnflcm_utils._get_vim(context,
             vnf_instance.vim_connection_info)
@@ -531,23 +566,30 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             context, vnf_instance.vnfd_id,
             vnf_instance.instantiated_vnf_info.flavour_id)
 
-        # TODO(LiangLu): grant_request and grant here is planned to
+        if vnf_dict['before_error_point'] <= EP.VNF_CONFIG_START:
-        # pass as a parameter, however due to they are not
+            # TODO(LiangLu): grant_request and grant here is planned to
-        # passed from conductor to vnflcm_driver, thus we put Null
+            # pass as a parameter, however due to they are not
-        # value to grant and grant_reqeust temporary.
+            # passed from conductor to vnflcm_driver, thus we put Null
-        # This part will be updated in next release.
+            # value to grant and grant_reqeust temporary.
-        self._mgmt_manager.invoke(
+            # This part will be updated in next release.
-            self._load_vnf_interface(
+            self._mgmt_manager.invoke(
-                context, 'terminate_start', vnf_instance, vnfd_dict),
+                self._load_vnf_interface(
-            'terminate_start', context=context,
+                    context, 'terminate_start', vnf_instance, vnfd_dict),
-            vnf_instance=vnf_instance,
+                'terminate_start', context=context,
-            terminate_vnf_request=terminate_vnf_req,
+                vnf_instance=vnf_instance,
-            grant=None, grant_request=None)
+                terminate_vnf_request=terminate_vnf_req,
+                grant=None, grant_request=None)
+
+        vnf_dict['current_error_point'] = EP.PRE_VIM_CONTROL
+
         LOG.info("Terminating vnf %s", vnf_instance.id)
         try:
-            self._delete_vnf_instance_resources(context, vnf_instance,
+            if vnf_dict['before_error_point'] <= EP.POST_VIM_CONTROL:
-                    vim_connection_info, terminate_vnf_req=terminate_vnf_req)
+                self._delete_vnf_instance_resources(context, vnf_instance,
+                        vim_connection_info,
+                        terminate_vnf_req=terminate_vnf_req)
 
+            vnf_dict['current_error_point'] = EP.INTERNAL_PROCESSING
             vnf_instance.instantiated_vnf_info.reinitialize()
             self._vnf_instance_update(context, vnf_instance,
                 vim_connection_info=[], task_state=None)
@@ -555,21 +597,31 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             LOG.info("Vnf terminated %s successfully", vnf_instance.id)
         except Exception as exp:
             with excutils.save_and_reraise_exception():
+                if vnf_dict['current_error_point'] == EP.PRE_VIM_CONTROL:
+                    if hasattr(vnf_instance.instantiated_vnf_info,
+                            'instance_id'):
+                        if vnf_instance.instantiated_vnf_info.instance_id:
+                            vnf_dict['current_error_point'] = \
+                                EP.POST_VIM_CONTROL
+
                 LOG.error("Unable to terminate vnf '%s' instance. "
                           "Error: %s", vnf_instance.id,
                           encodeutils.exception_to_unicode(exp))
-        # TODO(LiangLu): grant_request and grant here is planned to
+
-        # pass as a parameter, however due to they are not
+        vnf_dict['current_error_point'] = EP.VNF_CONFIG_END
-        # passed from conductor to vnflcm_driver, thus we put Null
+        if vnf_dict['before_error_point'] <= EP.VNF_CONFIG_END:
-        # value to grant and grant_reqeust temporary.
+            # TODO(LiangLu): grant_request and grant here is planned to
-        # This part will be updated in next release.
+            # pass as a parameter, however due to they are not
-        self._mgmt_manager.invoke(
+            # passed from conductor to vnflcm_driver, thus we put Null
-            self._load_vnf_interface(
+            # value to grant and grant_reqeust temporary.
-                context, 'terminate_end', vnf_instance, vnfd_dict),
+            # This part will be updated in next release.
-            'terminate_end', context=context,
+            self._mgmt_manager.invoke(
-            vnf_instance=vnf_instance,
+                self._load_vnf_interface(
-            terminate_vnf_request=terminate_vnf_req,
+                    context, 'terminate_end', vnf_instance, vnfd_dict),
-            grant=None, grant_request=None)
+                'terminate_end', context=context,
+                vnf_instance=vnf_instance,
+                terminate_vnf_request=terminate_vnf_req,
+                grant=None, grant_request=None)
 
     def _delete_vnf_instance_resources(self, context, vnf_instance,
             vim_connection_info, terminate_vnf_req=None,
@@ -614,7 +666,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             vnf_resource.destroy(context)
 
     def _heal_vnf(self, context, vnf_instance, vim_connection_info,
-            heal_vnf_request):
+            heal_vnf_request, vnf):
         inst_vnf_info = vnf_instance.instantiated_vnf_info
         try:
             self._vnf_manager.invoke(
@@ -631,6 +683,8 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                 raise exceptions.VnfHealFailed(id=vnf_instance.id,
                     error=encodeutils.exception_to_unicode(exp))
 
+        vnf['current_error_point'] = EP.POST_VIM_CONTROL
+
         try:
             self._vnf_manager.invoke(
                 vim_connection_info.vim_type, 'heal_vnf_wait',
@@ -665,19 +719,21 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
 
     def _respawn_vnf(self, context, vnf_instance, vnf_dict,
                     vim_connection_info, heal_vnf_request):
-        try:
+        if vnf_dict['before_error_point'] != EP.POST_VIM_CONTROL:
-            self._delete_vnf_instance_resources(context, vnf_instance,
+            try:
-                vim_connection_info, update_instantiated_state=False)
+                self._delete_vnf_instance_resources(context, vnf_instance,
-        except Exception as exc:
+                    vim_connection_info, update_instantiated_state=False)
-            with excutils.save_and_reraise_exception() as exc_ctxt:
+            except Exception as exc:
-                exc_ctxt.reraise = False
+                with excutils.save_and_reraise_exception() as exc_ctxt:
-                err_msg = ("Failed to delete vnf resources for vnf instance "
+                    exc_ctxt.reraise = False
-                          "%(id)s before respawning. The vnf is in "
+                    err_msg = ("Failed to delete vnf resources for "
-                          "inconsistent state. Error: %(error)s")
+                              "vnf instance %(id)s before respawning. "
-                LOG.error(err_msg % {"id": vnf_instance.id,
+                              "The vnf is in inconsistent state. "
-                          "error": str(exc)})
+                              "Error: %(error)s")
-                raise exceptions.VnfHealFailed(id=vnf_instance.id,
+                    LOG.error(err_msg % {"id": vnf_instance.id,
-                    error=encodeutils.exception_to_unicode(exc))
+                              "error": str(exc)})
+                    raise exceptions.VnfHealFailed(id=vnf_instance.id,
+                        error=encodeutils.exception_to_unicode(exc))
 
         # InstantiateVnfRequest is not stored in the db as it's mapped
         # to InstantiatedVnfInfo version object. Convert InstantiatedVnfInfo
@@ -721,6 +777,8 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
     @log.log
     @revert_to_error_task_state
     def heal_vnf(self, context, vnf_instance, vnf_dict, heal_vnf_request):
+        vnf_dict['current_error_point'] = EP.VNF_CONFIG_START
+
         LOG.info("Request received for healing vnf '%s'", vnf_instance.id)
         vim_info = vnflcm_utils._get_vim(context,
             vnf_instance.vim_connection_info)
@@ -731,47 +789,76 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
         vnfd_dict = vnflcm_utils._get_vnfd_dict(
             context, vnf_instance.vnfd_id,
             vnf_instance.instantiated_vnf_info.flavour_id)
-        # TODO(LiangLu): grant_request here is planned to pass
-        # as a parameter, however due to grant_request are not
-        # passed from conductor to vnflcm_driver, thus we put Null
-        # value to grant and grant_reqeust temporary.
-        # This part will be updated in next release.
-        self._mgmt_manager.invoke(
-            self._load_vnf_interface(
-                context, 'heal_start', vnf_instance, vnfd_dict),
-            'heal_start', context=context,
-            vnf_instance=vnf_instance,
-            heal_vnf_request=heal_vnf_request,
-            grant=vnf_dict.get('grant'), grant_request=None)
 
-        if not heal_vnf_request.vnfc_instance_id:
+        if vnf_dict['before_error_point'] <= EP.VNF_CONFIG_START:
-            self._respawn_vnf(context, vnf_instance, vnf_dict,
+            # TODO(LiangLu): grant_request here is planned to pass
-                             vim_connection_info, heal_vnf_request)
+            # as a parameter, however due to grant_request are not
-        else:
+            # passed from conductor to vnflcm_driver, thus we put Null
-            self._heal_vnf(context, vnf_instance, vim_connection_info,
+            # value to grant and grant_reqeust temporary.
-                      heal_vnf_request)
+            # This part will be updated in next release.
+            self._mgmt_manager.invoke(
+                self._load_vnf_interface(
+                    context, 'heal_start', vnf_instance, vnfd_dict),
+                'heal_start', context=context,
+                vnf_instance=vnf_instance,
+                heal_vnf_request=heal_vnf_request,
+                grant=vnf_dict.get('grant'), grant_request=None)
 
-        LOG.info("Request received for healing vnf '%s' is completed "
+        vnf_dict['current_error_point'] = EP.PRE_VIM_CONTROL
-                 "successfully", vnf_instance.id)
+
-        # TODO(LiangLu): grant_request here is planned to pass
+        try:
-        # as a parameter, however due to grant_request are not
+            heal_flag = False
-        # passed from conductor to vnflcm_driver, thus we put Null
+            if vnf_dict['before_error_point'] <= EP.POST_VIM_CONTROL:
-        # value to grant and grant_reqeust temporary.
+                if not heal_vnf_request.vnfc_instance_id:
-        # This part will be updated in next release.
+                    self._respawn_vnf(context, vnf_instance, vnf_dict,
-        self._mgmt_manager.invoke(
+                                    vim_connection_info, heal_vnf_request)
-            self._load_vnf_interface(
+                else:
-                context, 'heal_end', vnf_instance, vnfd_dict),
+                    heal_flag = True
-            'heal_end', context=context,
+                    self._heal_vnf(context, vnf_instance, vim_connection_info,
-            vnf_instance=vnf_instance,
+                            heal_vnf_request, vnf_dict)
-            heal_vnf_request=heal_vnf_request,
+
-            grant=vnf_dict.get('grant'), grant_request=None)
+                LOG.info("Request received for healing vnf '%s' is completed "
+                        "successfully", vnf_instance.id)
+        except Exception as exp:
+            with excutils.save_and_reraise_exception():
+                if vnf_dict['current_error_point'] == EP.PRE_VIM_CONTROL:
+                    if not heal_flag:
+                        if hasattr(vnf_instance.instantiated_vnf_info,
+                                'instance_id'):
+                            if vnf_instance.instantiated_vnf_info.instance_id:
+                                vnf_dict['current_error_point'] = \
+                                    EP.POST_VIM_CONTROL
+
+                LOG.error("Unable to heal vnf '%s' instance. "
+                          "Error: %s", heal_vnf_request.vnfc_instance_id,
+                          encodeutils.exception_to_unicode(exp))
+
+                raise exceptions.VnfHealFailed(id=vnf_instance.id,
+                    error=encodeutils.exception_to_unicode(exp))
+
+        vnf_dict['current_error_point'] = EP.VNF_CONFIG_END
+
+        if vnf_dict['before_error_point'] <= EP.VNF_CONFIG_END:
+            # TODO(LiangLu): grant_request here is planned to pass
+            # as a parameter, however due to grant_request are not
+            # passed from conductor to vnflcm_driver, thus we put Null
+            # value to grant and grant_reqeust temporary.
+            # This part will be updated in next release.
+            self._mgmt_manager.invoke(
+                self._load_vnf_interface(
+                    context, 'heal_end', vnf_instance, vnfd_dict),
+                'heal_end', context=context,
+                vnf_instance=vnf_instance,
+                heal_vnf_request=heal_vnf_request,
+                grant=vnf_dict.get('grant'), grant_request=None)
 
     def _scale_vnf_pre(self, context, vnf_info, vnf_instance,
                       scale_vnf_request, vim_connection_info):
-        self._vnfm_plugin._update_vnf_scaling(
+        if vnf_info['before_error_point'] <= EP.NOTIFY_PROCESSING:
-            context, vnf_info, 'ACTIVE', 'PENDING_' + scale_vnf_request.type)
+            self._vnfm_plugin._update_vnf_scaling(
-        vnf_lcm_op_occ = vnf_info['vnf_lcm_op_occ']
+                context, vnf_info, 'ACTIVE', 'PENDING_'
-        vnf_lcm_op_occ.error_point = 2
+                + scale_vnf_request.type)
+        vnf_info['current_error_point'] = EP.VNF_CONFIG_START
 
         scale_id_list = []
         scale_name_list = []
@@ -798,21 +885,23 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                     number_of_steps=scale_vnf_request.number_of_steps
                 )
             vnf_info['res_num'] = res_num
-            # TODO(LiangLu): grant_request here is planned to pass
+
-            # as a parameter, however due to grant_request are not
+            if vnf_info['before_error_point'] <= EP.VNF_CONFIG_START:
-            # passed from conductor to vnflcm_driver, thus we put Null
+                # TODO(LiangLu): grant_request here is planned to pass
-            # value to grant and grant_reqeust temporary.
+                # as a parameter, however due to grant_request are not
-            # This part will be updated in next release.
+                # passed from conductor to vnflcm_driver, thus we put Null
-            if len(scale_id_list) != 0:
+                # value to grant and grant_reqeust temporary.
-                kwargs = {'scale_name_list': scale_name_list}
+                # This part will be updated in next release.
-                self._mgmt_manager.invoke(
+                if len(scale_id_list) != 0:
-                    self._load_vnf_interface(
+                    kwargs = {'scale_name_list': scale_name_list}
-                        context, 'scale_start', vnf_instance, vnfd_dict),
+                    self._mgmt_manager.invoke(
-                    'scale_start', context=context,
+                        self._load_vnf_interface(
-                    vnf_instance=vnf_instance,
+                            context, 'scale_start', vnf_instance, vnfd_dict),
-                    scale_vnf_request=scale_vnf_request,
+                        'scale_start', context=context,
-                    grant=vnf_info.get('grant'), grant_request=None,
+                        vnf_instance=vnf_instance,
-                    **kwargs)
+                        scale_vnf_request=scale_vnf_request,
+                        grant=vnf_info.get('grant'), grant_request=None,
+                        **kwargs)
         else:
             vnf_info['action'] = 'out'
             scale_id_list = self._vnf_manager.invoke(
@@ -824,7 +913,8 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                 auth_attr=vim_connection_info.access_info,
                 region_name=vim_connection_info.access_info.get('region_name')
             )
-        vnf_lcm_op_occ.error_point = 3
+
+        vnf_info['current_error_point'] = EP.PRE_VIM_CONTROL
         return scale_id_list, scale_name_list, grp_id
 
     def _get_node_template_for_vnf(self, vnfd_dict):
@@ -841,7 +931,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                        scale_id_list,
                        resource_changes):
         vnf_lcm_op_occ = vnf_info['vnf_lcm_op_occ']
-        vnf_lcm_op_occ.error_point = 6
+        vnf_info['current_error_point'] = EP.VNF_CONFIG_END
         if scale_vnf_request.type == 'SCALE_OUT':
             vnfd_dict = vnflcm_utils._get_vnfd_dict(
                 context, vnf_instance.vnfd_id,
@@ -859,22 +949,24 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             id_list = []
             id_list = list(set(scale_id_after) - set(scale_id_list))
             vnf_info['res_num'] = len(scale_id_after)
-            # TODO(LiangLu): grant_request here is planned to pass
+
-            # as a parameter, however due to grant_request are not
+            if vnf_info['before_error_point'] <= EP.VNF_CONFIG_END:
-            # passed from conductor to vnflcm_driver, thus we put Null
+                # TODO(LiangLu): grant_request here is planned to pass
-            # value to grant and grant_reqeust temporary.
+                # as a parameter, however due to grant_request are not
-            # This part will be updated in next release.
+                # passed from conductor to vnflcm_driver, thus we put Null
-            if len(id_list) != 0:
+                # value to grant and grant_reqeust temporary.
-                kwargs = {'scale_out_id_list': id_list}
+                # This part will be updated in next release.
-                self._mgmt_manager.invoke(
+                if len(id_list) != 0:
-                    self._load_vnf_interface(
+                    kwargs = {'scale_out_id_list': id_list}
-                        context, 'scale_end', vnf_instance, vnfd_dict),
+                    self._mgmt_manager.invoke(
-                    'scale_end', context=context,
+                        self._load_vnf_interface(
-                    vnf_instance=vnf_instance,
+                            context, 'scale_end', vnf_instance, vnfd_dict),
-                    scale_vnf_request=scale_vnf_request,
+                        'scale_end', context=context,
-                    grant=vnf_info.get('grant'), grant_request=None,
+                        vnf_instance=vnf_instance,
-                    **kwargs)
+                        scale_vnf_request=scale_vnf_request,
-        vnf_lcm_op_occ.error_point = 7
+                        grant=vnf_info.get('grant'), grant_request=None,
+                        **kwargs)
+
         vnf_instance.instantiated_vnf_info.scale_level =\
             vnf_info['after_scale_level']
         if vim_connection_info.vim_type != 'kubernetes':
@@ -888,15 +980,18 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                 vnf_info['res_num']
             vnf_info['attributes']['scale_group'] =\
                 jsonutils.dump_as_bytes(scaleGroupDict)
-        vnf_lcm_op_occ = vnf_info['vnf_lcm_op_occ']
+
-        vnf_lcm_op_occ.operation_state = 'COMPLETED'
+        if vnf_info['before_error_point'] < EP.NOTIFY_COMPLETED:
-        vnf_lcm_op_occ.resource_changes = resource_changes
+            vnf_lcm_op_occ = vnf_info['vnf_lcm_op_occ']
-        vnf_instance.task_state = None
+            vnf_lcm_op_occ.operation_state = 'COMPLETED'
-        self._vnfm_plugin._update_vnf_scaling(context, vnf_info,
+            vnf_lcm_op_occ.resource_changes = resource_changes
-                                          'PENDING_' + scale_vnf_request.type,
+            vnf_instance.task_state = None
-                                          'ACTIVE',
+            self._vnfm_plugin._update_vnf_scaling(context, vnf_info,
-                                          vnf_instance=vnf_instance,
+                                    'PENDING_' + scale_vnf_request.type,
-                                          vnf_lcm_op_occ=vnf_lcm_op_occ)
+                                    'ACTIVE', vnf_instance=vnf_instance,
+                                    vnf_lcm_op_occ=vnf_lcm_op_occ)
+
+        vnf_info['current_error_point'] = EP.NOTIFY_COMPLETED
 
         notification = vnf_info['notification']
         notification['notificationStatus'] = 'RESULT'
@@ -1073,11 +1168,8 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
         # action_driver
         LOG.debug("vnf_info['vnfd']['attributes'] %s",
         vnf_info['vnfd']['attributes'])
-        vnf_lcm_op_occ = vnf_info['vnf_lcm_op_occ']
-        vnf_lcm_op_occ.error_point = 4
         self.scale(context, vnf_info, scale_vnf_request,
                    vim_connection_info, scale_name_list, grp_id)
-        vnf_lcm_op_occ.error_point = 5
 
     @log.log
     @revert_to_error_scale
@@ -1096,6 +1188,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
         notification['operationState'] = 'PROCESSING'
         self.rpc_api.send_notification(context, notification)
 
+        vnf_info['current_error_point'] = EP.NOTIFY_PROCESSING
         vim_info = vnflcm_utils._get_vim(context,
             vnf_instance.vim_connection_info)
 
@@ -1108,17 +1201,20 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             scale_vnf_request,
             vim_connection_info)
 
-        self._scale_vnf(context, vnf_info,
+        if vnf_info['before_error_point'] <= EP.POST_VIM_CONTROL:
-                        vnf_instance,
+            self._scale_vnf(context, vnf_info,
-                        scale_vnf_request,
+                            vnf_instance,
-                        vim_connection_info,
+                            scale_vnf_request,
-                        scale_name_list, grp_id)
+                            vim_connection_info,
+                            scale_name_list, grp_id)
 
-        resource_changes = self._scale_resource_update(context, vnf_info,
+            resource_changes = self._scale_resource_update(context, vnf_info,
                                     vnf_instance,
                                     scale_vnf_request,
                                     vim_connection_info)
 
+        vnf_info['current_error_point'] = EP.INTERNAL_PROCESSING
+
         self._scale_vnf_post(context, vnf_info,
                              vnf_instance,
                              scale_vnf_request,
@@ -1192,6 +1288,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                 scale_name_list=scale_name_list,
                 grp_id=grp_id
             )
+            vnf_info['current_error_point'] = EP.POST_VIM_CONTROL
             self._vnf_manager.invoke(
                 vim_connection_info.vim_type,
                 'scale_update_wait',
@@ -1212,6 +1309,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                 scale_vnf_request=scale_vnf_request,
                 region_name=vim_connection_info.access_info.get('region_name')
             )
+            vnf_info['current_error_point'] = EP.POST_VIM_CONTROL
             self._vnf_manager.invoke(
                 vim_connection_info.vim_type,
                 'scale_update_wait',
@@ -1259,6 +1357,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                     region_name=vim_connection_info.access_info.get('\
                         region_name')
                 )
+                vnf_info['current_error_point'] = EP.POST_VIM_CONTROL
                 self._vnf_manager.invoke(
                     vim_connection_info.vim_type,
                     'scale_wait',
@@ -1402,7 +1501,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                     vnf_info['after_scale_level'] = scale.scale_level
                     break
         if vnf_lcm_op_occs.operation == 'SCALE' \
-                and vnf_lcm_op_occs.error_point >= 4:
+           and vnf_lcm_op_occs.error_point >= EP.POST_VIM_CONTROL:
             scale_id_list, scale_name_list, grp_id = self._vnf_manager.invoke(
                 vim_connection_info.vim_type,
                 'get_rollback_ids',
@@ -1413,7 +1512,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                 auth_attr=vim_connection_info.access_info,
                 region_name=vim_connection_info.access_info.get('region_name')
             )
-        if vnf_lcm_op_occs.error_point == 7:
+        if vnf_lcm_op_occs.error_point == EP.NOTIFY_COMPLETED:
             if vnf_lcm_op_occs.operation == 'SCALE':
                 vnfd_dict = vnflcm_utils._get_vnfd_dict(
                     context, vnf_instance.vnfd_id,
@@ -1454,7 +1553,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                         vnf_instance=vnf_instance,
                         terminate_vnf_request=None,
                         grant=None, grant_request=None)
-            vnf_lcm_op_occs.error_point = 6
+            vnf_lcm_op_occs.error_point = EP.VNF_CONFIG_END
 
         return scale_name_list, grp_id
 
@@ -1468,7 +1567,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             scale_name_list,
             grp_id):
         vnf_lcm_op_occs = vnf_info['vnf_lcm_op_occ']
-        if vnf_lcm_op_occs.error_point >= 4:
+        if vnf_lcm_op_occs.error_point >= EP.POST_VIM_CONTROL:
             if vnf_lcm_op_occs.operation == 'SCALE':
                 scale_vnf_request = objects.ScaleVnfRequest.obj_from_primitive(
                     operation_params, context=context)
@@ -1505,7 +1604,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
                     'delete_wait', plugin=self, context=context,
                     vnf_id=instance_id, auth_attr=access_info)
 
-        vnf_lcm_op_occs.error_point = 3
+        vnf_lcm_op_occs.error_point = EP.PRE_VIM_CONTROL
 
     def _update_vnf_rollback_pre(self, context, vnf_info):
         self._vnfm_plugin._update_vnf_rollback_pre(context, vnf_info)
@@ -1548,7 +1647,7 @@ class VnfLcmDriver(abstract_driver.VnfInstanceAbstractDriver):
             resource_changes = self._term_resource_update(
                 context, vnf_info, vnf_instance)
 
-        vnf_lcm_op_occs.error_point = 2
+        vnf_lcm_op_occs.error_point = EP.VNF_CONFIG_START
 
         timestamp = datetime.utcnow()
         vnf_lcm_op_occs.operation_state = 'ROLLED_BACK'

tacker/vnfm/infra_drivers/openstack/heat_client.py

@@ -39,6 +39,12 @@ class HeatClient(object):
                 for x in self._stack_ids(stack.id):
                     yield x
 
+    def find_stack(self, **kwargs):
+        stack = None
+        for sub_stack in self.stacks.list(**{"filters": kwargs}):
+            stack = sub_stack
+        return stack
+
     def create(self, fields):
         fields = fields.copy()
         fields['disable_rollback'] = True

tacker/vnfm/infra_drivers/openstack/openstack.py

@@ -297,18 +297,67 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
             for name, value in nested_hot_dict.items():
                 vnf['attributes'].update({name: self._format_base_hot(value)})
 
-            vnf['error_point'] = 4
+            stack = None
-            # Create heat-stack with BaseHOT and parameters
+            if 'before_error_point' not in vnf or \
-            stack = self._create_stack_with_user_data(
+               vnf['before_error_point'] <= fields.ErrorPoint.PRE_VIM_CONTROL:
-                heatclient, vnf, base_hot_dict,
+                # Create heat-stack with BaseHOT and parameters
-                nested_hot_dict, hot_param_dict)
+                stack = self._create_stack_with_user_data(
+                    heatclient, vnf, base_hot_dict,
+                    nested_hot_dict, hot_param_dict)
+            elif vnf['before_error_point'] == \
+                    fields.ErrorPoint.POST_VIM_CONTROL:
+                stack_found = None
+                try:
+                    # Find existing stack
+                    filters = {"name": "vnflcm_" + vnf["id"]}
+                    stack_found = heatclient.find_stack(**filters)
+                except Exception as exc:
+                    LOG.error("Stack name %s was not found due to %s",
+                        ("vnflcm_" + vnf["id"]), str(exc))
+
+                if stack_found:
+                    # Update heat-stack with BaseHOT and parameters
+                    self._update_stack_with_user_data(heatclient, vnf,
+                                                    base_hot_dict,
+                                                    nested_hot_dict,
+                                                    hot_param_dict,
+                                                    stack_found.id)
+                    return stack_found.id
 
         elif user_data_path is None and user_data_class is None:
             LOG.info('Execute heat-translator and create heat-stack.')
             tth = translate_template.TOSCAToHOT(vnf, heatclient,
                                                 inst_req_info, grant_info)
             tth.generate_hot()
-            stack = self._create_stack(heatclient, tth.vnf, tth.fields)
+            stack = None
+            if 'before_error_point' not in vnf or \
+               vnf['before_error_point'] <= fields.ErrorPoint.PRE_VIM_CONTROL:
+                stack = self._create_stack(heatclient, tth.vnf, tth.fields)
+            elif vnf['before_error_point'] == \
+                    fields.ErrorPoint.POST_VIM_CONTROL:
+                stack_found = None
+                try:
+                    # Find existing stack
+                    name_filter = None
+                    if 'stack_name' in vnf['attributes'].keys():
+                        name_filter = vnf['attributes']['stack_name']
+                    else:
+                        name_filter = (vnf['name'].replace(' ', '_')
+                            + '_' + vnf['id'])
+                        if vnf['attributes'].get('failure_count'):
+                            name_filter += ('-RESPAWN-%s') \
+                                % str(vnf['attributes']['failure_count'])
+                    filters = {"name": name_filter}
+                    stack_found = heatclient.find_stack(**filters)
+                except Exception as exc:
+                    LOG.error("Stack name %s was not found due to %s",
+                        name_filter, str(exc))
+
+                if stack_found:
+                    # Update heat-stack
+                    self._update_stack(heatclient, stack_found.id, tth.fields)
+
+                    return stack_found.id
         else:
             error_reason = _(
                 "failed to get lcm-operation-user-data or "
@@ -331,6 +380,26 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
             mp_list = mp_list[0:-1]
         LOG.debug('Remove sys.modules: %s', sys.modules)
 
+    @log.log
+    def _update_stack_with_user_data(self, heatclient, vnf,
+                                     base_hot_dict, nested_hot_dict,
+                                     hot_param_dict, stack_id):
+        try:
+            stack_update_param = {
+                'parameters': hot_param_dict,
+                'template': self._format_base_hot(base_hot_dict),
+                'stack_name': ("vnflcm_" + vnf["id"]),
+                'timeout_mins': (
+                    self.STACK_RETRIES * self.STACK_RETRY_WAIT // 60)}
+            if nested_hot_dict:
+                files_dict = {}
+                for name, value in nested_hot_dict.items():
+                    files_dict[name] = self._format_base_hot(value)
+                stack_update_param['files'] = files_dict
+            heatclient.update(stack_id, **stack_update_param)
+        except Exception as exc:
+            LOG.error("Error during update due to %s", str(exc))
+
     @log.log
     def _create_stack_with_user_data(self, heatclient, vnf,
                                      base_hot_dict, nested_hot_dict,
@@ -360,6 +429,13 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
 
         return yaml.safe_dump(base_hot_dict)
 
+    @log.log
+    def _update_stack(self, heatclient, stack_id, fields):
+        fields['timeout_mins'] = (
+            self.STACK_RETRIES * self.STACK_RETRY_WAIT // 60)
+
+        heatclient.update(stack_id, **fields)
+
     @log.log
     def _create_stack(self, heatclient, vnf, fields):
         if 'stack_name' not in fields:
@@ -380,6 +456,21 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
 
         return stack
 
+    @log.log
+    def update_stack_wait(self, plugin, context, vnf_dict, stack_id,
+            auth_attr):
+        """Check stack is updateded successfully"""
+
+        region_name = vnf_dict.get('placement_attr', {}).get(
+            'region_name', None)
+
+        stack = self._wait_until_stack_ready(stack_id,
+            auth_attr, infra_cnst.STACK_UPDATE_IN_PROGRESS,
+            infra_cnst.STACK_UPDATE_COMPLETE,
+            vnfm.VNFUpdateWaitFailed, region_name=region_name)
+
+        return stack
+
     @log.log
     def create_wait(self, plugin, context, vnf_dict, vnf_id, auth_attr):
         region_name = vnf_dict.get('placement_attr', {}).get(
@@ -1130,6 +1221,8 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
         region_name = access_info.get('region')
         inst_vnf_info = vnf_instance.instantiated_vnf_info
         heatclient = hc.HeatClient(access_info, region_name=region_name)
+        vnf_lcm_op_occs = objects.VnfLcmOpOcc.get_by_vnf_instance_id(
+            context, vnf_instance.id)
 
         def _get_storage_resources(vnfc_resource):
             # Prepare list of storage resources to be marked unhealthy
@@ -1212,8 +1305,9 @@ class OpenStack(abstract_driver.VnfAbstractDriver,
                     "statuses": ",".join(stack_statuses),
                     "status": stack.stack_status})
 
-        _get_stack_status()
+        if vnf_lcm_op_occs.error_point <= fields.ErrorPoint.PRE_VIM_CONTROL:
-        _resource_mark_unhealthy()
+            _get_stack_status()
+            _resource_mark_unhealthy()
 
         LOG.info("Updating stack %(stack)s for vnf instance %(id)s",
                 {"stack": inst_vnf_info.instance_id, "id": vnf_instance.id})