From 96d4ac6353b674ff8b7d9a8e7246b18b8222c8a1 Mon Sep 17 00:00:00 2001 From: Yi Feng <fengyi@fujitsu.com> Date: Fri, 20 Jan 2023 12:59:36 +0900 Subject: [PATCH] Add support VNFM auto heal and scale Added two new interfaces to Prometheus Plugin. Tacker as a VNFM can support AutoHeal and AutoScale operations for VNFs and CNFs through External Monitoring Tools, without NFVO. Implements: blueprint support-auto-lcm Change-Id: Ib0b5fd9264d80b9666ce69190e0ee41bbde23fac --- .zuul.yaml | 4 + etc/tacker/api-paste.ini | 6 +- ...heal-scale-with-vnfm-2f9a4533b12c96ed.yaml | 9 + .../sol_refactored/api/policies/vnfpm_v2.py | 21 +- .../api/prometheus_plugin_router.py | 12 +- .../api/schemas/prometheus_plugin_schemas.py | 3 +- tacker/sol_refactored/common/config.py | 19 + .../common/prometheus_plugin.py | 167 +++++--- tacker/sol_refactored/common/vnflcm_utils.py | 142 +++++++ .../conductor/conductor_rpc_v2.py | 14 +- .../sol_refactored/conductor/conductor_v2.py | 13 +- .../conductor/prometheus_plugin_driver.py | 98 ++++- .../prometheus_plugin_controller.py | 16 +- tacker/sol_refactored/controller/vnflcm_v2.py | 114 +----- .../functional/sol_kubernetes_v2/base_v2.py | 14 +- .../functional/sol_kubernetes_v2/paramgen.py | 35 ++ .../test_prom_auto_scale_heal.py | 355 +++++++++++++++++ .../test_prometheus_auto_scaling.py | 115 ------ .../sol_v2/test_prom_auto_scale_heal.py | 361 ++++++++++++++++++ .../tests/functional/sol_v2_common/base_v2.py | 10 + .../functional/sol_v2_common/paramgen.py | 71 ++++ .../common/test_prometheus_plugin.py | 34 ++ .../conductor/test_prometheus_plugin.py | 89 ++++- .../controller/test_prometheus_plugin.py | 167 +++++++- 24 files changed, 1570 insertions(+), 319 deletions(-) create mode 100644 releasenotes/notes/support-auto-heal-scale-with-vnfm-2f9a4533b12c96ed.yaml create mode 100644 tacker/sol_refactored/common/vnflcm_utils.py create mode 100644 tacker/tests/functional/sol_kubernetes_v2/test_prom_auto_scale_heal.py delete mode 100644 tacker/tests/functional/sol_kubernetes_v2/test_prometheus_auto_scaling.py create mode 100644 tacker/tests/functional/sol_v2/test_prom_auto_scale_heal.py diff --git a/.zuul.yaml b/.zuul.yaml index ef42b7870..8fc3261cd 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -299,6 +299,9 @@ $TACKER_CONF: server_notification: server_notification: true + prometheus_plugin: + auto_scaling: true + auto_healing: true tox_envlist: dsvm-functional-sol-v2 - job: @@ -608,6 +611,7 @@ fault_management: true performance_management: true auto_scaling: true + auto_healing: true test_rule_with_promtool: true tox_envlist: dsvm-functional-sol-kubernetes-v2 vars: diff --git a/etc/tacker/api-paste.ini b/etc/tacker/api-paste.ini index 6e4de6679..f1bf03438 100644 --- a/etc/tacker/api-paste.ini +++ b/etc/tacker/api-paste.ini @@ -8,7 +8,8 @@ use = egg:Paste#urlmap /vnflcm/v2: vnflcm_v2 /vnffm/v1: vnffm_v1 /vnfpm/v2: vnfpm_v2 -/alert/vnf_instances: prometheus_auto_scaling +/alert/auto_scaling: prometheus_auto_scaling +/alert/auto_healing: prometheus_auto_healing /alert: prometheus_fm /pm_event: prometheus_pm /server_notification: server_notification @@ -93,6 +94,9 @@ paste.app_factory = tacker.sol_refactored.api.router:VnffmAPIRouterV1.factory [app:prometheus_auto_scaling] paste.app_factory = tacker.sol_refactored.api.prometheus_plugin_router:AutoScalingRouter.factory +[app:prometheus_auto_healing] +paste.app_factory = tacker.sol_refactored.api.prometheus_plugin_router:AutoHealingRouter.factory + [app:prometheus_fm] paste.app_factory = tacker.sol_refactored.api.prometheus_plugin_router:FmAlertRouter.factory diff --git a/releasenotes/notes/support-auto-heal-scale-with-vnfm-2f9a4533b12c96ed.yaml b/releasenotes/notes/support-auto-heal-scale-with-vnfm-2f9a4533b12c96ed.yaml new file mode 100644 index 000000000..20a6e998d --- /dev/null +++ b/releasenotes/notes/support-auto-heal-scale-with-vnfm-2f9a4533b12c96ed.yaml @@ -0,0 +1,9 @@ +--- +features: + - | + In Prometheus Plugin, two new interfaces are added to implement AutoHeal + and AutoScale for VNF and CNF. + As a VNFM, Tacker can decide whether to perform heal and scale operations + through the received alarms sent by External Monitoring Tools, without + NFVO. + Added new user guide to help users understand the function. diff --git a/tacker/sol_refactored/api/policies/vnfpm_v2.py b/tacker/sol_refactored/api/policies/vnfpm_v2.py index a81d1d269..b1c9ca3bf 100644 --- a/tacker/sol_refactored/api/policies/vnfpm_v2.py +++ b/tacker/sol_refactored/api/policies/vnfpm_v2.py @@ -27,7 +27,8 @@ REPORT_GET = '/vnfpm/v2/pm_jobs/{id}/reports/{report_id}' POLICY_NAME_PROM_PLUGIN = 'tacker_PROM_PLUGIN_api:PROM_PLUGIN:{}' PROM_PLUGIN_PM_PATH = '/pm_event' -PROM_PLUGIN_AUTO_SCALING_PATH = '/alert/vnf_instances' +PROM_PLUGIN_AUTO_HEALING_PATH = '/alert/auto_healing' +PROM_PLUGIN_AUTO_SCALING_PATH = '/alert/auto_scaling' rules = [ policy.DocumentedRuleDefault( @@ -107,6 +108,15 @@ rules = [ 'path': PROM_PLUGIN_PM_PATH} ] ), + policy.DocumentedRuleDefault( + name=POLICY_NAME_PROM_PLUGIN.format('auto_healing'), + check_str=RULE_ANY, + description="auto_healing", + operations=[ + {'method': 'POST', + 'path': PROM_PLUGIN_AUTO_HEALING_PATH} + ] + ), policy.DocumentedRuleDefault( name=POLICY_NAME_PROM_PLUGIN.format('auto_scaling'), check_str=RULE_ANY, @@ -115,15 +125,6 @@ rules = [ {'method': 'POST', 'path': PROM_PLUGIN_AUTO_SCALING_PATH} ] - ), - policy.DocumentedRuleDefault( - name=POLICY_NAME_PROM_PLUGIN.format('auto_scaling_id'), - check_str=RULE_ANY, - description="auto_scaling_id", - operations=[ - {'method': 'POST', - 'path': PROM_PLUGIN_AUTO_SCALING_PATH + '/{vnfInstanceId}'} - ] ) ] diff --git a/tacker/sol_refactored/api/prometheus_plugin_router.py b/tacker/sol_refactored/api/prometheus_plugin_router.py index 6a44fe5e4..a52da1deb 100644 --- a/tacker/sol_refactored/api/prometheus_plugin_router.py +++ b/tacker/sol_refactored/api/prometheus_plugin_router.py @@ -33,11 +33,15 @@ class FmAlertRouter(prom_wsgi.PrometheusPluginAPIRouter): route_list = [("", {"POST": "alert"})] +class AutoHealingRouter(prom_wsgi.PrometheusPluginAPIRouter): + controller = prom_wsgi.PrometheusPluginResource( + prometheus_plugin_controller.AutoHealingController(), + policy_name=vnfpm_policy_v2.POLICY_NAME_PROM_PLUGIN) + route_list = [("", {"POST": "auto_healing"})] + + class AutoScalingRouter(prom_wsgi.PrometheusPluginAPIRouter): controller = prom_wsgi.PrometheusPluginResource( prometheus_plugin_controller.AutoScalingController(), policy_name=vnfpm_policy_v2.POLICY_NAME_PROM_PLUGIN) - route_list = [ - ("", {"POST": "auto_scaling"}), - ("/{id}", {"POST": "auto_scaling"}) - ] + route_list = [("", {"POST": "auto_scaling"})] diff --git a/tacker/sol_refactored/api/schemas/prometheus_plugin_schemas.py b/tacker/sol_refactored/api/schemas/prometheus_plugin_schemas.py index cd27b0eae..a44df9cbc 100644 --- a/tacker/sol_refactored/api/schemas/prometheus_plugin_schemas.py +++ b/tacker/sol_refactored/api/schemas/prometheus_plugin_schemas.py @@ -34,11 +34,12 @@ Alert = { }, 'function_type': { 'type': 'string', - 'enum': ['vnffm', 'vnfpm', 'auto_scale'] + 'enum': ['vnffm', 'vnfpm', 'auto_scale', 'auto_heal'] }, 'job_id': {'type': 'string'}, 'object_instance_id': {'type': 'string'}, 'vnf_instance_id': {'type': 'string'}, + 'vnfc_info_id': {'type': 'string'}, 'node': {'type': 'string'}, 'perceived_severity': { 'type': 'string', diff --git a/tacker/sol_refactored/common/config.py b/tacker/sol_refactored/common/config.py index a8e163fdd..a9dde3071 100644 --- a/tacker/sol_refactored/common/config.py +++ b/tacker/sol_refactored/common/config.py @@ -162,6 +162,9 @@ PROMETHEUS_PLUGIN_OPTS = [ cfg.BoolOpt('fault_management', default=False, help=_('Enable prometheus plugin fault management')), + cfg.BoolOpt('auto_healing', + default=False, + help=_('Enable prometheus plugin autohealing')), cfg.BoolOpt('auto_scaling', default=False, help=_('Enable prometheus plugin autoscaling')), @@ -189,6 +192,22 @@ PROMETHEUS_PLUGIN_OPTS = [ 'This configuration is changed in case of replacing ' 'the original function with a vendor specific ' 'function.')), + cfg.StrOpt('auto_healing_package', + default='tacker.sol_refactored.common.prometheus_plugin', + help=_('Package name for auto healing. ' + 'This configuration is changed in case of replacing ' + 'the original function with a vendor specific ' + 'function.')), + cfg.StrOpt('auto_healing_class', + default='PrometheusPluginAutoHealing', + help=_('Class name for auto healing. ' + 'This configuration is changed in case of replacing ' + 'the original function with a vendor specific ' + 'function.')), + cfg.IntOpt('timer_interval', + default=20, + help=_('Timeout (second) of packing for multiple ' + 'auto healing.')), cfg.StrOpt('auto_scaling_package', default='tacker.sol_refactored.common.prometheus_plugin', help=_('Package name for auto scaling. ' diff --git a/tacker/sol_refactored/common/prometheus_plugin.py b/tacker/sol_refactored/common/prometheus_plugin.py index f33093ac9..0c08d2742 100644 --- a/tacker/sol_refactored/common/prometheus_plugin.py +++ b/tacker/sol_refactored/common/prometheus_plugin.py @@ -145,7 +145,7 @@ class PrometheusPluginPm(PrometheusPlugin, mon_base.MonitoringPlugin): self._alert(kwargs['request'], body=kwargs['body']) except Exception as e: # All exceptions is ignored here and 204 response will always - # be returned. Because when tacker responds error to alertmanager, + # be returned because when tacker responds error to alertmanager, # alertmanager may repeat the same reports. LOG.error("%s: %s", e.__class__.__name__, e.args[0]) @@ -237,16 +237,16 @@ class PrometheusPluginPm(PrometheusPlugin, mon_base.MonitoringPlugin): result = [] context = request.context datetime_now = datetime.datetime.now(datetime.timezone.utc) - for alt in body['alerts']: - if alt['labels']['function_type'] != 'vnfpm': + for alert in body['alerts']: + if alert['labels']['function_type'] != 'vnfpm': continue try: - pm_job_id = alt['labels']['job_id'] - object_instance_id = alt['labels']['object_instance_id'] - metric = alt['labels']['metric'] - sub_object_instance_id = alt['labels'].get( + pm_job_id = alert['labels']['job_id'] + object_instance_id = alert['labels']['object_instance_id'] + metric = alert['labels']['metric'] + sub_object_instance_id = alert['labels'].get( 'sub_object_instance_id') - value = alt['annotations']['value'] + value = alert['annotations']['value'] pm_job = pm_job_utils.get_pm_job(context, pm_job_id) self.filter_alert_by_time(context, pm_job, datetime_now, @@ -700,7 +700,7 @@ class PrometheusPluginFm(PrometheusPlugin, mon_base.MonitoringPlugin): self._alert(kwargs['request'], body=kwargs['body']) except Exception as e: # All exceptions is ignored here and 204 response will always - # be returned. Because when tacker responds error to alertmanager, + # be returned because when tacker responds error to alertmanager, # alertmanager may repeat the same reports. LOG.error("%s: %s", e.__class__.__name__, e.args[0]) @@ -831,18 +831,91 @@ class PrometheusPluginFm(PrometheusPlugin, mon_base.MonitoringPlugin): def _alert(self, request, body): now = datetime.datetime.now(datetime.timezone.utc) result = [] - for alt in body['alerts']: - if alt['labels']['function_type'] != 'vnffm': + for alert in body['alerts']: + if alert['labels']['function_type'] != 'vnffm': continue try: alarms = self.create_or_update_alarm( - request.context, alt, now) + request.context, alert, now) result.extend(alarms) except sol_ex.PrometheusPluginSkipped: pass return result +class PrometheusPluginAutoHealing(PrometheusPlugin, mon_base.MonitoringPlugin): + _instance = None + + @staticmethod + def instance(): + if PrometheusPluginAutoHealing._instance is None: + if not CONF.prometheus_plugin.auto_healing: + stub = mon_base.MonitoringPluginStub.instance() + PrometheusPluginAutoHealing._instance = stub + else: + PrometheusPluginAutoHealing() + return PrometheusPluginAutoHealing._instance + + def __init__(self): + if PrometheusPluginAutoHealing._instance: + raise SystemError( + "Not constructor but instance() should be used.") + super(PrometheusPluginAutoHealing, self).__init__() + self.set_callback(self.default_callback) + PrometheusPluginAutoHealing._instance = self + + def set_callback(self, notification_callback): + self.notification_callback = notification_callback + + def alert(self, **kwargs): + try: + self._alert(kwargs['request'], body=kwargs['body']) + except Exception as e: + # All exceptions is ignored here and 204 response will always + # be returned because when tacker responds error to alertmanager, + # alertmanager may repeat the same reports. + LOG.error("%s: %s", e.__class__.__name__, e.args[0]) + + def default_callback(self, context, vnf_instance_id, vnfc_info_id): + self.rpc.enqueue_auto_heal_instance( + context, vnf_instance_id, vnfc_info_id) + + @validator.schema(prometheus_plugin_schemas.AlertMessage) + def _alert(self, request, body): + context = request.context + alerts = (alert for alert in body['alerts'] if + alert['status'] == 'firing' and + alert['labels']['receiver_type'] == 'tacker' and + alert['labels']['function_type'] == 'auto_heal') + + for alert in alerts: + vnf_instance_id = alert['labels']['vnf_instance_id'] + try: + inst = inst_utils.get_inst(context, vnf_instance_id) + except sol_ex.VnfInstanceNotFound: + continue + if inst.instantiationState != 'INSTANTIATED': + self.rpc.dequeue_auto_heal_instance( + None, vnf_instance_id) + + if (not inst.obj_attr_is_set('vnfConfigurableProperties') + or not inst.vnfConfigurableProperties.get( + 'isAutohealEnabled')): + continue + + vnfc_info_id = alert['labels']['vnfc_info_id'] + result = { + vnfcInfo for vnfcInfo in inst.instantiatedVnfInfo.vnfcInfo + if vnfcInfo.id == vnfc_info_id + } + if not result: + continue + + if self.notification_callback: + self.notification_callback( + context, vnf_instance_id, vnfc_info_id) + + class PrometheusPluginAutoScaling(PrometheusPlugin, mon_base.MonitoringPlugin): _instance = None @@ -861,7 +934,7 @@ class PrometheusPluginAutoScaling(PrometheusPlugin, mon_base.MonitoringPlugin): raise SystemError( "Not constructor but instance() should be used.") super(PrometheusPluginAutoScaling, self).__init__() - self.notification_callback = self.default_callback + self.set_callback(self.default_callback) PrometheusPluginAutoScaling._instance = self def set_callback(self, notification_callback): @@ -872,46 +945,46 @@ class PrometheusPluginAutoScaling(PrometheusPlugin, mon_base.MonitoringPlugin): self._alert(kwargs['request'], body=kwargs['body']) except Exception as e: # All exceptions is ignored here and 204 response will always - # be returned. Because when tacker responds error to alertmanager, + # be returned because when tacker responds error to alertmanager, # alertmanager may repeat the same reports. LOG.error("%s: %s", e.__class__.__name__, e.args[0]) def default_callback(self, context, vnf_instance_id, scaling_param): - self.rpc.request_scale(context, vnf_instance_id, scaling_param) - - def skip_if_auto_scale_not_enabled(self, vnf_instance): - if (not vnf_instance.obj_attr_is_set('vnfConfigurableProperties') or - not vnf_instance.vnfConfigurableProperties.get( - 'isAutoscaleEnabled')): - raise sol_ex.PrometheusPluginSkipped() - - def process_auto_scale(self, request, vnf_instance_id, auto_scale_type, - aspect_id): - scaling_param = { - 'type': auto_scale_type, - 'aspectId': aspect_id, - } - context = request.context - if self.notification_callback: - self.notification_callback(context, vnf_instance_id, scaling_param) + self.rpc.trigger_scale(context, vnf_instance_id, scaling_param) @validator.schema(prometheus_plugin_schemas.AlertMessage) def _alert(self, request, body): - result = [] - for alt in body['alerts']: - if alt['labels']['function_type'] != 'auto_scale': - continue - try: - vnf_instance_id = alt['labels']['vnf_instance_id'] - auto_scale_type = alt['labels']['auto_scale_type'] - aspect_id = alt['labels']['aspect_id'] - context = request.context + context = request.context + alerts = (alert for alert in body['alerts'] if + alert['status'] == 'firing' and + alert['labels']['receiver_type'] == 'tacker' and + alert['labels']['function_type'] == 'auto_scale') + for alert in alerts: + vnf_instance_id = alert['labels']['vnf_instance_id'] + try: inst = inst_utils.get_inst(context, vnf_instance_id) - self.skip_if_auto_scale_not_enabled(inst) - self.process_auto_scale( - request, vnf_instance_id, auto_scale_type, aspect_id) - result.append((vnf_instance_id, auto_scale_type, aspect_id)) - except sol_ex.PrometheusPluginSkipped: - pass - return result + except sol_ex.VnfInstanceNotFound: + continue + if (inst.instantiationState != 'INSTANTIATED' or + not inst.obj_attr_is_set('vnfConfigurableProperties') or + not inst.vnfConfigurableProperties.get( + 'isAutoscaleEnabled')): + continue + + aspect_id = alert['labels']['aspect_id'] + result = { + scaleStatus for scaleStatus in + inst.instantiatedVnfInfo.scaleStatus + if scaleStatus.aspectId == aspect_id + } + if not result: + continue + + scaling_param = { + 'type': alert['labels']['auto_scale_type'], + 'aspectId': aspect_id, + } + if self.notification_callback: + self.notification_callback( + context, vnf_instance_id, scaling_param) diff --git a/tacker/sol_refactored/common/vnflcm_utils.py b/tacker/sol_refactored/common/vnflcm_utils.py new file mode 100644 index 000000000..c809ea0bc --- /dev/null +++ b/tacker/sol_refactored/common/vnflcm_utils.py @@ -0,0 +1,142 @@ +# Copyright (C) 2023 Fujitsu +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from datetime import datetime + +from oslo_utils import uuidutils + +from tacker.sol_refactored.common import coordinate +from tacker.sol_refactored.common import exceptions as sol_ex +from tacker.sol_refactored.common import lcm_op_occ_utils as lcmocc_utils +from tacker.sol_refactored.common import vnf_instance_utils as inst_utils +from tacker.sol_refactored.conductor import conductor_rpc_v2 +from tacker.sol_refactored import objects +from tacker.sol_refactored.objects.v2 import fields as v2fields + + +# TODO(fengyi): The code of this function is all copied from the +# controller, which is not friendly to future development and +# maintenance, and may be refactored in the future. After +# refactoring, only the validation of the req body is left, +# and the creation of lcmocc and the call to start_lcm_op are +# all executed by the controller, notification driver, etc. +@coordinate.lock_vnf_instance('{vnf_instance_id}') +def heal(context, vnf_instance_id, body): + inst = inst_utils.get_inst(context, vnf_instance_id) + + if inst.instantiationState != 'INSTANTIATED': + raise sol_ex.VnfInstanceIsNotInstantiated(inst_id=vnf_instance_id) + + lcmocc_utils.check_lcmocc_in_progress(context, vnf_instance_id) + + # check parameter for later use + is_all = body.get('additionalParams', {}).get('all', False) + if not isinstance(is_all, bool): + raise sol_ex.SolValidationError( + detail="additionalParams['all'] must be bool.") + + if 'vnfcInstanceId' in body: + inst_info = inst.instantiatedVnfInfo + vnfc_id = [] + if inst_info.obj_attr_is_set('vnfcInfo'): + vnfc_id = [vnfc.id for vnfc in inst_info.vnfcInfo] + for req_vnfc_id in body['vnfcInstanceId']: + if req_vnfc_id not in vnfc_id: + raise sol_ex.SolValidationError( + detail="vnfcInstanceId(%s) does not exist." + % req_vnfc_id) + + lcmocc = new_lcmocc(vnf_instance_id, v2fields.LcmOperationType.HEAL, body) + lcmocc.create(context) + + rpc = conductor_rpc_v2.VnfLcmRpcApiV2() + rpc.start_lcm_op(context, lcmocc.id) + return lcmocc + + +# TODO(fengyi): The code of this function is all copied from the +# controller, which is not friendly to future development and +# maintenance, and may be refactored in the future. After +# refactoring, only the validation of the req body is left, +# and the creation of lcmocc and the call to start_lcm_op are +# all executed by the controller, notification driver, etc. +@coordinate.lock_vnf_instance('{vnf_instance_id}') +def scale(context, vnf_instance_id, body): + inst = inst_utils.get_inst(context, vnf_instance_id) + + if inst.instantiationState != 'INSTANTIATED': + raise sol_ex.VnfInstanceIsNotInstantiated(inst_id=vnf_instance_id) + + lcmocc_utils.check_lcmocc_in_progress(context, vnf_instance_id) + + # check parameters + aspect_id = body['aspectId'] + if 'numberOfSteps' not in body: + # set default value (1) defined by SOL specification for + # the convenience of the following methods. + body['numberOfSteps'] = 1 + + scale_level = _get_current_scale_level(inst, aspect_id) + max_scale_level = _get_max_scale_level(inst, aspect_id) + if scale_level is None or max_scale_level is None: + raise sol_ex.InvalidScaleAspectId(aspect_id=aspect_id) + + num_steps = body['numberOfSteps'] + if body['type'] == 'SCALE_IN': + num_steps *= -1 + scale_level += num_steps + if scale_level < 0 or scale_level > max_scale_level: + raise sol_ex.InvalidScaleNumberOfSteps( + num_steps=body['numberOfSteps']) + + lcmocc = new_lcmocc(vnf_instance_id, v2fields.LcmOperationType.SCALE, body) + lcmocc.create(context) + + rpc = conductor_rpc_v2.VnfLcmRpcApiV2() + rpc.start_lcm_op(context, lcmocc.id) + return lcmocc + + +def _get_current_scale_level(inst, aspect_id): + if (inst.obj_attr_is_set('instantiatedVnfInfo') and + inst.instantiatedVnfInfo.obj_attr_is_set('scaleStatus')): + for scale_info in inst.instantiatedVnfInfo.scaleStatus: + if scale_info.aspectId == aspect_id: + return scale_info.scaleLevel + + +def _get_max_scale_level(inst, aspect_id): + if (inst.obj_attr_is_set('instantiatedVnfInfo') and + inst.instantiatedVnfInfo.obj_attr_is_set('maxScaleLevels')): + for scale_info in inst.instantiatedVnfInfo.maxScaleLevels: + if scale_info.aspectId == aspect_id: + return scale_info.scaleLevel + + +def new_lcmocc(inst_id, operation, req_body, + op_state=v2fields.LcmOperationStateType.STARTING): + now = datetime.utcnow() + lcmocc = objects.VnfLcmOpOccV2( + id=uuidutils.generate_uuid(), + operationState=op_state, + stateEnteredTime=now, + startTime=now, + vnfInstanceId=inst_id, + operation=operation, + isAutomaticInvocation=False, + isCancelPending=False, + operationParams=req_body) + + return lcmocc diff --git a/tacker/sol_refactored/conductor/conductor_rpc_v2.py b/tacker/sol_refactored/conductor/conductor_rpc_v2.py index 7826433b8..437cfe09c 100644 --- a/tacker/sol_refactored/conductor/conductor_rpc_v2.py +++ b/tacker/sol_refactored/conductor/conductor_rpc_v2.py @@ -95,5 +95,15 @@ class PrometheusPluginConductor(object): def store_job_info(self, context, report): self.cast(context, 'store_job_info', report=report) - def request_scale(self, context, id, scale_req): - self.cast(context, 'request_scale', id=id, scale_req=scale_req) + def trigger_scale(self, context, id, scale_req): + self.cast(context, 'trigger_scale', id=id, scale_req=scale_req) + + def enqueue_auto_heal_instance( + self, context, vnf_instance_id, vnfc_info_id): + self.cast(context, 'enqueue_auto_heal_instance', + vnf_instance_id=vnf_instance_id, + vnfc_info_id=vnfc_info_id) + + def dequeue_auto_heal_instance(self, context, vnf_instance_id): + self.cast(context, 'dequeue_auto_heal_instance', + vnf_instance_id=vnf_instance_id) diff --git a/tacker/sol_refactored/conductor/conductor_v2.py b/tacker/sol_refactored/conductor/conductor_v2.py index fbbf7ebe8..7b3aac3b1 100644 --- a/tacker/sol_refactored/conductor/conductor_v2.py +++ b/tacker/sol_refactored/conductor/conductor_v2.py @@ -394,8 +394,17 @@ class ConductorV2(object): self.vnfpm_driver.store_job_info(context, report) @log.log - def request_scale(self, context, id, scale_req): - self.prom_driver.request_scale(context, id, scale_req) + def trigger_scale(self, context, id, scale_req): + self.prom_driver.trigger_scale(context, id, scale_req) + + @log.log + def enqueue_auto_heal_instance( + self, context, vnf_instance_id, vnfc_info_id): + self.prom_driver.enqueue_heal(context, vnf_instance_id, vnfc_info_id) + + @log.log + def dequeue_auto_heal_instance(self, context, vnf_instance_id): + self.prom_driver.dequeue_heal(vnf_instance_id) @log.log def server_notification_notify( diff --git a/tacker/sol_refactored/conductor/prometheus_plugin_driver.py b/tacker/sol_refactored/conductor/prometheus_plugin_driver.py index 7a93480d1..c61d0f3bd 100644 --- a/tacker/sol_refactored/conductor/prometheus_plugin_driver.py +++ b/tacker/sol_refactored/conductor/prometheus_plugin_driver.py @@ -13,17 +13,66 @@ # License for the specific language governing permissions and limitations # under the License. +import threading + from oslo_log import log as logging from tacker.sol_refactored.common import config as cfg -from tacker.sol_refactored.common import http_client +from tacker.sol_refactored.common import vnflcm_utils +from tacker.sol_refactored import objects LOG = logging.getLogger(__name__) CONF = cfg.CONF +class VnfmAutoHealTimer(): + def __init__(self, context, vnf_instance_id, + expiration_time, expiration_handler): + self.lock = threading.Lock() + self.expired = False + self.queue = set() + self.context = context + self.vnf_instance_id = vnf_instance_id + self.expiration_handler = expiration_handler + self.timer = threading.Timer(expiration_time, self.expire) + self.timer.start() + + def expire(self): + _expired = False + with self.lock: + if not self.expired: + self._cancel() + _expired = True + if _expired: + self.expiration_handler( + self.context, self.vnf_instance_id, list(self.queue)) + + def add_vnfc_info_id(self, vnfc_info_id): + with self.lock: + if not self.expired: + self.queue.add(vnfc_info_id) + + def _cancel(self): + self.timer.cancel() + self.expired = True + + def cancel(self): + with self.lock: + if not self.expired: + self._cancel() + + def __del__(self): + self.cancel() + + class PrometheusPluginDriverStub(): - def request_scale(self, context, vnf_instance_id, scale_req): + def trigger_scale(self, context, vnf_instance_id, scale_req): + pass + + def enqueue_heal(self, context, vnf_instance_id, vnfc_info_id): + pass + + def dequeue_heal(self, vnf_instance_id): pass @@ -34,6 +83,7 @@ class PrometheusPluginDriver(): def instance(): if PrometheusPluginDriver._instance is None: if (CONF.prometheus_plugin.auto_scaling or + CONF.prometheus_plugin.auto_healing or CONF.prometheus_plugin.fault_management or CONF.prometheus_plugin.performance_management): PrometheusPluginDriver() @@ -45,19 +95,35 @@ class PrometheusPluginDriver(): def __init__(self): if PrometheusPluginDriver._instance: raise SystemError("Not constructor but instance() should be used.") - auth_handle = http_client.KeystonePasswordAuthHandle( - auth_url=CONF.keystone_authtoken.auth_url, - username=CONF.keystone_authtoken.username, - password=CONF.keystone_authtoken.password, - project_name=CONF.keystone_authtoken.project_name, - user_domain_name=CONF.keystone_authtoken.user_domain_name, - project_domain_name=CONF.keystone_authtoken.project_domain_name) - self.client = http_client.HttpClient(auth_handle) PrometheusPluginDriver._instance = self + self.timer_map = {} + self.expiration_time = CONF.prometheus_plugin.timer_interval - def request_scale(self, context, vnf_instance_id, scale_req): - ep = CONF.v2_vnfm.endpoint - url = f'{ep}/vnflcm/v2/vnf_instances/{vnf_instance_id}/scale' - resp, _ = self.client.do_request( - url, "POST", context=context, body=scale_req, version="2.0.0") - LOG.info("AutoScaling request is processed: %d.", resp.status_code) + def enqueue_heal(self, context, vnf_instance_id, vnfc_info_id): + if vnf_instance_id not in self.timer_map: + self.timer_map[vnf_instance_id] = VnfmAutoHealTimer( + context, vnf_instance_id, self.expiration_time, + self._timer_expired) + self.timer_map[vnf_instance_id].add_vnfc_info_id(vnfc_info_id) + + def dequeue_heal(self, vnf_instance_id): + if vnf_instance_id in self.timer_map: + self.timer_map[vnf_instance_id].cancel() + del self.timer_map[vnf_instance_id] + + def _trigger_heal(self, context, vnf_instance_id, vnfc_info_ids): + heal_req = objects.HealVnfRequest(vnfcInstanceId=vnfc_info_ids) + body = heal_req.to_dict() + LOG.info(f"VNFM AutoHealing is triggered. vnf: {vnf_instance_id}, " + f"vnfcInstanceId: {vnfc_info_ids}") + vnflcm_utils.heal(context, vnf_instance_id, body) + + def _timer_expired(self, context, vnf_instance_id, vnfc_info_ids): + self.dequeue_heal(vnf_instance_id) + self._trigger_heal(context, vnf_instance_id, vnfc_info_ids) + + def trigger_scale(self, context, vnf_instance_id, scale_req): + LOG.info(f"VNFM AutoScaling is triggered. vnf: {vnf_instance_id}, " + f"type: {scale_req['type']}, aspectId: " + f"{scale_req['aspectId']}") + vnflcm_utils.scale(context, vnf_instance_id, scale_req) diff --git a/tacker/sol_refactored/controller/prometheus_plugin_controller.py b/tacker/sol_refactored/controller/prometheus_plugin_controller.py index 0da23661c..e6c3f3d4b 100644 --- a/tacker/sol_refactored/controller/prometheus_plugin_controller.py +++ b/tacker/sol_refactored/controller/prometheus_plugin_controller.py @@ -47,6 +47,19 @@ class FmAlertController(prom_wsgi.PrometheusPluginAPIController): return prom_wsgi.PrometheusPluginResponse(204, None) +class AutoHealingController(prom_wsgi.PrometheusPluginAPIController): + def auto_healing(self, request, body): + if not CONF.prometheus_plugin.auto_healing: + raise sol_ex.PrometheusPluginNotEnabled( + name='Auto healing') + cls = mon_base.get_class( + CONF.prometheus_plugin.auto_healing_package, + CONF.prometheus_plugin.auto_healing_class) + mon_base.MonitoringPlugin.get_instance(cls).alert( + request=request, body=body) + return prom_wsgi.PrometheusPluginResponse(204, None) + + class AutoScalingController(prom_wsgi.PrometheusPluginAPIController): def auto_scaling(self, request, body): if not CONF.prometheus_plugin.auto_scaling: @@ -58,6 +71,3 @@ class AutoScalingController(prom_wsgi.PrometheusPluginAPIController): mon_base.MonitoringPlugin.get_instance(cls).alert( request=request, body=body) return prom_wsgi.PrometheusPluginResponse(204, None) - - def auto_scaling_id(self, request, _, body): - return self.auto_scaling(request, body) diff --git a/tacker/sol_refactored/controller/vnflcm_v2.py b/tacker/sol_refactored/controller/vnflcm_v2.py index fbf574b72..9040e8348 100644 --- a/tacker/sol_refactored/controller/vnflcm_v2.py +++ b/tacker/sol_refactored/controller/vnflcm_v2.py @@ -14,8 +14,6 @@ # under the License. -from datetime import datetime - from oslo_log import log as logging from oslo_utils import uuidutils @@ -30,6 +28,7 @@ from tacker.sol_refactored.common import lcm_op_occ_utils as lcmocc_utils from tacker.sol_refactored.common import subscription_utils as subsc_utils from tacker.sol_refactored.common import vim_utils from tacker.sol_refactored.common import vnf_instance_utils as inst_utils +from tacker.sol_refactored.common import vnflcm_utils from tacker.sol_refactored.conductor import conductor_rpc_v2 from tacker.sol_refactored.controller import vnflcm_view from tacker.sol_refactored.nfvo import nfvo_client @@ -153,22 +152,6 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController): self.endpoint) return sol_wsgi.SolResponse(204, None) - def _new_lcmocc(self, inst_id, operation, req_body, - op_state=v2fields.LcmOperationStateType.STARTING): - now = datetime.utcnow() - lcmocc = objects.VnfLcmOpOccV2( - id=uuidutils.generate_uuid(), - operationState=op_state, - stateEnteredTime=now, - startTime=now, - vnfInstanceId=inst_id, - operation=operation, - isAutomaticInvocation=False, - isCancelPending=False, - operationParams=req_body) - - return lcmocc - @validator.schema(schema.VnfInfoModificationRequest_V200, '2.0.0') @coordinate.lock_vnf_instance('{id}') def update(self, request, id, body): @@ -205,7 +188,8 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController): detail="vnfcInstanceId(%s) does not exist." % vnfc_mod['id']) - lcmocc = self._new_lcmocc(id, v2fields.LcmOperationType.MODIFY_INFO, + lcmocc = vnflcm_utils.new_lcmocc( + id, v2fields.LcmOperationType.MODIFY_INFO, body, v2fields.LcmOperationStateType.PROCESSING) lcmocc.create(context) @@ -226,8 +210,8 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController): lcmocc_utils.check_lcmocc_in_progress(context, id) - lcmocc = self._new_lcmocc(id, v2fields.LcmOperationType.INSTANTIATE, - body) + lcmocc = vnflcm_utils.new_lcmocc( + id, v2fields.LcmOperationType.INSTANTIATE, body) req_param = lcmocc.operationParams # if there is partial vimConnectionInfo check and fulfill here. @@ -260,8 +244,8 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController): lcmocc_utils.check_lcmocc_in_progress(context, id) - lcmocc = self._new_lcmocc(id, v2fields.LcmOperationType.TERMINATE, - body) + lcmocc = vnflcm_utils.new_lcmocc( + id, v2fields.LcmOperationType.TERMINATE, body) lcmocc.create(context) self.conductor_rpc.start_lcm_op(context, lcmocc.id) @@ -270,93 +254,19 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController): return sol_wsgi.SolResponse(202, None, location=location) - def _get_current_scale_level(self, inst, aspect_id): - if (inst.obj_attr_is_set('instantiatedVnfInfo') and - inst.instantiatedVnfInfo.obj_attr_is_set('scaleStatus')): - for scale_info in inst.instantiatedVnfInfo.scaleStatus: - if scale_info.aspectId == aspect_id: - return scale_info.scaleLevel - - def _get_max_scale_level(self, inst, aspect_id): - if (inst.obj_attr_is_set('instantiatedVnfInfo') and - inst.instantiatedVnfInfo.obj_attr_is_set('maxScaleLevels')): - for scale_info in inst.instantiatedVnfInfo.maxScaleLevels: - if scale_info.aspectId == aspect_id: - return scale_info.scaleLevel - @validator.schema(schema.ScaleVnfRequest_V200, '2.0.0') - @coordinate.lock_vnf_instance('{id}') def scale(self, request, id, body): context = request.context - inst = inst_utils.get_inst(context, id) - - if inst.instantiationState != 'INSTANTIATED': - raise sol_ex.VnfInstanceIsNotInstantiated(inst_id=id) - - lcmocc_utils.check_lcmocc_in_progress(context, id) - - # check parameters - aspect_id = body['aspectId'] - if 'numberOfSteps' not in body: - # set default value (1) defined by SOL specification for - # the convenience of the following methods. - body['numberOfSteps'] = 1 - - scale_level = self._get_current_scale_level(inst, aspect_id) - max_scale_level = self._get_max_scale_level(inst, aspect_id) - if scale_level is None or max_scale_level is None: - raise sol_ex.InvalidScaleAspectId(aspect_id=aspect_id) - - num_steps = body['numberOfSteps'] - if body['type'] == 'SCALE_IN': - num_steps *= -1 - scale_level += num_steps - if scale_level < 0 or scale_level > max_scale_level: - raise sol_ex.InvalidScaleNumberOfSteps( - num_steps=body['numberOfSteps']) - - lcmocc = self._new_lcmocc(id, v2fields.LcmOperationType.SCALE, - body) - lcmocc.create(context) - - self.conductor_rpc.start_lcm_op(context, lcmocc.id) + lcmocc = vnflcm_utils.scale(context, id, body) location = lcmocc_utils.lcmocc_href(lcmocc.id, self.endpoint) return sol_wsgi.SolResponse(202, None, location=location) @validator.schema(schema.HealVnfRequest_V200, '2.0.0') - @coordinate.lock_vnf_instance('{id}') def heal(self, request, id, body): context = request.context - inst = inst_utils.get_inst(context, id) - - if inst.instantiationState != 'INSTANTIATED': - raise sol_ex.VnfInstanceIsNotInstantiated(inst_id=id) - - lcmocc_utils.check_lcmocc_in_progress(context, id) - - # check parameter for later use - is_all = body.get('additionalParams', {}).get('all', False) - if not isinstance(is_all, bool): - raise sol_ex.SolValidationError( - detail="additionalParams['all'] must be bool.") - - if 'vnfcInstanceId' in body: - inst_info = inst.instantiatedVnfInfo - vnfc_id = [] - if inst_info.obj_attr_is_set('vnfcInfo'): - vnfc_id = [vnfc.id for vnfc in inst_info.vnfcInfo] - for req_vnfc_id in body['vnfcInstanceId']: - if req_vnfc_id not in vnfc_id: - raise sol_ex.SolValidationError( - detail="vnfcInstanceId(%s) does not exist." - % req_vnfc_id) - - lcmocc = self._new_lcmocc(id, v2fields.LcmOperationType.HEAL, body) - lcmocc.create(context) - - self.conductor_rpc.start_lcm_op(context, lcmocc.id) + lcmocc = vnflcm_utils.heal(context, id, body) location = lcmocc_utils.lcmocc_href(lcmocc.id, self.endpoint) @@ -373,7 +283,7 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController): lcmocc_utils.check_lcmocc_in_progress(context, id) - lcmocc = self._new_lcmocc( + lcmocc = vnflcm_utils.new_lcmocc( id, v2fields.LcmOperationType.CHANGE_EXT_CONN, body) lcmocc.create(context) @@ -458,8 +368,8 @@ class VnfLcmControllerV2(sol_wsgi.SolAPIController): raise sol_ex.SolValidationError( detail="'lcm-kubernetes-def-files' must be specified") - lcmocc = self._new_lcmocc(id, v2fields.LcmOperationType.CHANGE_VNFPKG, - body) + lcmocc = vnflcm_utils.new_lcmocc( + id, v2fields.LcmOperationType.CHANGE_VNFPKG, body) lcmocc.create(context) diff --git a/tacker/tests/functional/sol_kubernetes_v2/base_v2.py b/tacker/tests/functional/sol_kubernetes_v2/base_v2.py index ec4f57b2d..ac831ad4c 100644 --- a/tacker/tests/functional/sol_kubernetes_v2/base_v2.py +++ b/tacker/tests/functional/sol_kubernetes_v2/base_v2.py @@ -288,6 +288,13 @@ class BaseVnfLcmKubernetesV2Test(base.BaseTestCase): return self.tacker_client.do_request( path, "GET", version="2.0.0") + def list_lcmocc(self, filter_expr=None): + path = "/vnflcm/v2/vnf_lcm_op_occs" + if filter_expr: + path = "{}?{}".format(path, urllib.parse.urlencode(filter_expr)) + return self.tacker_client.do_request( + path, "GET", version="2.0.0") + def create_subscription(self, req_body): path = "/vnffm/v1/subscriptions" return self.tacker_client.do_request( @@ -370,7 +377,12 @@ class BaseVnfLcmKubernetesV2Test(base.BaseTestCase): path, "DELETE", version="2.1.0") def prometheus_auto_scaling_alert(self, req_body): - path = "/alert/vnf_instances" + path = "/alert/auto_scaling" + return self.tacker_client.do_request( + path, "POST", body=req_body) + + def prometheus_auto_healing_alert(self, req_body): + path = "/alert/auto_healing" return self.tacker_client.do_request( path, "POST", body=req_body) diff --git a/tacker/tests/functional/sol_kubernetes_v2/paramgen.py b/tacker/tests/functional/sol_kubernetes_v2/paramgen.py index 093f14912..a96596e88 100644 --- a/tacker/tests/functional/sol_kubernetes_v2/paramgen.py +++ b/tacker/tests/functional/sol_kubernetes_v2/paramgen.py @@ -739,3 +739,38 @@ def prometheus_auto_scaling_alert(inst_id): "groupKey": "{}:{}", "truncatedAlerts": 0 } + + +def prometheus_auto_healing_alert(inst_id, vnfc_info_id): + return { + "receiver": "receiver", + "status": "firing", + "alerts": [{ + "status": "firing", + "labels": { + "receiver_type": "tacker", + "function_type": "auto_heal", + "vnf_instance_id": inst_id, + "vnfc_info_id": vnfc_info_id + }, + "annotations": { + }, + "startsAt": "2022-06-21T23:47:36.453Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://controller147:9090/graph?g0.expr=" + "up%7Bjob%3D%22node%22%7D+%3D%3D+0&g0.tab=1", + "fingerprint": "5ef77f1f8a3ecb8d" + }], + "groupLabels": {}, + "commonLabels": { + "alertname": "NodeInstanceDown", + "job": "node" + }, + "commonAnnotations": { + "description": "sample" + }, + "externalURL": "http://controller147:9093", + "version": "4", + "groupKey": "{}:{}", + "truncatedAlerts": 0 + } diff --git a/tacker/tests/functional/sol_kubernetes_v2/test_prom_auto_scale_heal.py b/tacker/tests/functional/sol_kubernetes_v2/test_prom_auto_scale_heal.py new file mode 100644 index 000000000..700c4f6b6 --- /dev/null +++ b/tacker/tests/functional/sol_kubernetes_v2/test_prom_auto_scale_heal.py @@ -0,0 +1,355 @@ +# Copyright (C) 2023 Fujitsu +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ddt +import os +import time + +from tacker.tests.functional.sol_kubernetes_v2 import base_v2 +from tacker.tests.functional.sol_kubernetes_v2 import paramgen + +# Waiting time to trigger autoheal (unit: second) +WAIT_AUTO_HEAL_TIME = 23 + +# Waiting time to lcmocc update in DB (unit: second) +WAIT_LCMOCC_UPDATE_TIME = 3 + + +@ddt.ddt +class PromAutoScaleHealTest(base_v2.BaseVnfLcmKubernetesV2Test): + + @classmethod + def setUpClass(cls): + super(PromAutoScaleHealTest, cls).setUpClass() + + cur_dir = os.path.dirname(__file__) + + test_instantiate_cnf_resources_path = os.path.join( + cur_dir, "samples/test_instantiate_cnf_resources") + cls.vnf_pkg_1, cls.vnfd_id_1 = cls.create_vnf_package( + test_instantiate_cnf_resources_path) + + @classmethod + def tearDownClass(cls): + super(PromAutoScaleHealTest, cls).tearDownClass() + + cls.delete_vnf_package(cls.vnf_pkg_1) + + def setUp(self): + super(PromAutoScaleHealTest, self).setUp() + + def test_vnfm_auto_heal_cnf(self): + """Test Prometheus VNFM Auto Healing operations + + * About LCM operations: + This test includes the following operations. + - 1. Create a new VNF instance resource + - 2. Instantiate a VNF instance + - 3. Show OpOcc + - 4-5. Receive Alert and Auto Heal + - 6. Show OpOcc + - 7. Receive Alert + - 8-9. Receive Alert and Auto Heal + - 10. Show OpOcc + - 11. Terminate a VNF instance + - 12. Show OpOcc + - 13. Delete a VNF instance + """ + + # 1. LCM-Create: Create a new VNF instance resource + create_req = paramgen.instantiate_cnf_resources_create(self.vnfd_id_1) + resp, body = self.create_vnf_instance(create_req) + self.assertEqual(201, resp.status_code) + inst_id = body['id'] + + # 2. LCM-Instantiate: Instantiate a VNF instance + vim_id = self.get_k8s_vim_id() + instantiate_req = paramgen.min_sample_instantiate(vim_id) + instantiate_req['additionalParams'][ + 'lcm-kubernetes-def-files'] = ['Files/kubernetes/deployment.yaml'] + instantiate_req['vnfConfigurableProperties'] = { + 'isAutohealEnabled': True} + resp, body = self.instantiate_vnf_instance(inst_id, instantiate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # 3. LCM-Show-OpOccV2: Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('INSTANTIATE', body['operation']) + + # 4-5. Send alert and auto heal + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + vnfc_info_id = (affected_vnfcs[0]['vduId'] + '-' + + affected_vnfcs[0]['id']) + alert = paramgen.prometheus_auto_healing_alert(inst_id, vnfc_info_id) + resp, body = self.prometheus_auto_healing_alert(alert) + self.assertEqual(204, resp.status_code) + + # Since auto heal takes 20 seconds to trigger, + # wait 23 seconds here. + time.sleep(WAIT_AUTO_HEAL_TIME) + + # 6. LCM-Show-OpOccV2: Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + heal_lcmocc = [ + heal_lcmocc for heal_lcmocc in body + if heal_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = heal_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('HEAL', body['operation']) + self.assertEqual(2, len(affected_vnfcs)) + + added_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'ADDED'] + self.assertEqual(1, len(added_vnfcs)) + + removed_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'REMOVED'] + self.assertEqual(1, len(removed_vnfcs)) + + removed_vnfc_info_id = (affected_vnfcs[0]['vduId'] + '-' + + affected_vnfcs[0]['id']) + self.assertEqual(vnfc_info_id, removed_vnfc_info_id) + + # 7. Send alert + resp, body = self.show_vnf_instance(inst_id) + self.assertEqual(200, resp.status_code) + + vnfc_infos = body['instantiatedVnfInfo']['vnfcInfo'] + vnfc_info_id_1 = vnfc_infos[0]['id'] + alert = paramgen.prometheus_auto_healing_alert( + inst_id, vnfc_info_id_1) + resp, body = self.prometheus_auto_healing_alert(alert) + self.assertEqual(204, resp.status_code) + + # 8-9. Send alert and auto heal + vnfc_info_id_2 = vnfc_infos[1]['id'] + alert = paramgen.prometheus_auto_healing_alert( + inst_id, vnfc_info_id_2) + resp, body = self.prometheus_auto_healing_alert(alert) + self.assertEqual(204, resp.status_code) + + # Since auto heal takes 20 seconds to trigger, + # wait 23 seconds here. + time.sleep(WAIT_AUTO_HEAL_TIME) + + # 10. LCM-Show-OpOccV2: Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + heal_lcmocc = [ + heal_lcmocc for heal_lcmocc in body + if heal_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = heal_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('HEAL', body['operation']) + self.assertEqual(4, len(affected_vnfcs)) + + added_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'ADDED'] + self.assertEqual(2, len(added_vnfcs)) + + removed_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'REMOVED'] + self.assertEqual(2, len(removed_vnfcs)) + + removed_vnfc_info_ids = [ + removed_vnfcs[0]['vduId'] + '-' + removed_vnfcs[0]['id'], + removed_vnfcs[1]['vduId'] + '-' + removed_vnfcs[1]['id'] + ] + self.assertCountEqual( + [vnfc_info_id_1, vnfc_info_id_2], removed_vnfc_info_ids) + + # 11. LCM-Terminate: Terminate VNF + terminate_req = paramgen.terminate_vnf_min() + resp, body = self.terminate_vnf_instance(inst_id, terminate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and terminate completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 12. LCM-Show-OpOccV2: Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('TERMINATE', body['operation']) + + # 13. LCM-Delete: Delete a VNF instance + resp, body = self.delete_vnf_instance(inst_id) + self.assertEqual(204, resp.status_code) + + # check deletion of VNF instance + resp, body = self.show_vnf_instance(inst_id) + self.assertEqual(404, resp.status_code) + + def test_vnfm_auto_scale_cnf(self): + """Test Prometheus VNFM Auto Scaling operations + + * About LCM operations: + This test includes the following operations. + - 1. Create a new VNF instance resource + - 2. Instantiate a VNF instance + - 3. Show OpOcc + - 4-5. Receive Alert and Auto Scale out + - 6. Show OpOcc + - 7-8. Receive Alert and Auto Scale in + - 9. Show OpOcc + - 10. Terminate a VNF instance + - 11. Show OpOcc + - 12. Delete a VNF instance + """ + + # 1. LCM-Create: Create a new VNF instance resource + create_req = paramgen.instantiate_cnf_resources_create(self.vnfd_id_1) + resp, body = self.create_vnf_instance(create_req) + self.assertEqual(201, resp.status_code) + inst_id = body['id'] + + # 2. LCM-Instantiate: Instantiate a VNF instance + vim_id = self.get_k8s_vim_id() + instantiate_req = paramgen.min_sample_instantiate(vim_id) + instantiate_req['additionalParams'][ + 'lcm-kubernetes-def-files'] = ['Files/kubernetes/deployment.yaml'] + instantiate_req['vnfConfigurableProperties'] = { + 'isAutoscaleEnabled': True} + resp, body = self.instantiate_vnf_instance(inst_id, instantiate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # 3. LCM-Show-OpOccV2: Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('INSTANTIATE', body['operation']) + + # 4-5. Send alert and auto-scale out + alert = paramgen.prometheus_auto_scaling_alert(inst_id) + resp, body = self.prometheus_auto_scaling_alert(alert) + self.assertEqual(204, resp.status_code) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and scale completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 6. LCM-Show-OpOccV2: Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + scale_lcmocc = [ + scale_lcmocc for scale_lcmocc in body + if scale_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = scale_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('SCALE', body['operation']) + self.assertEqual(1, len(affected_vnfcs)) + self.assertEqual('ADDED', affected_vnfcs[0]['changeType']) + self.assertEqual( + alert['alerts'][0]['labels']['aspect_id'], + body['operationParams']['aspectId']) + + # 7-8. Send alert and auto-scale in + alert['alerts'][0]['labels']['auto_scale_type'] = 'SCALE_IN' + resp, body = self.prometheus_auto_scaling_alert(alert) + self.assertEqual(204, resp.status_code) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and scale completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 9. LCM-Show-OpOccV2: Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + scale_lcmocc = [ + scale_lcmocc for scale_lcmocc in body + if scale_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = scale_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('SCALE', body['operation']) + self.assertEqual(1, len(affected_vnfcs)) + self.assertEqual('REMOVED', affected_vnfcs[0]['changeType']) + self.assertEqual( + alert['alerts'][0]['labels']['aspect_id'], + body['operationParams']['aspectId']) + + # 10. LCM-Terminate: Terminate VNF + terminate_req = paramgen.terminate_vnf_min() + resp, body = self.terminate_vnf_instance(inst_id, terminate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and terminate completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 11. LCM-Show-OpOccV2: Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('TERMINATE', body['operation']) + + # 12. LCM-Delete: Delete a VNF instance + resp, body = self.delete_vnf_instance(inst_id) + self.assertEqual(204, resp.status_code) + + # check deletion of VNF instance + resp, body = self.show_vnf_instance(inst_id) + self.assertEqual(404, resp.status_code) diff --git a/tacker/tests/functional/sol_kubernetes_v2/test_prometheus_auto_scaling.py b/tacker/tests/functional/sol_kubernetes_v2/test_prometheus_auto_scaling.py deleted file mode 100644 index 784fbf86a..000000000 --- a/tacker/tests/functional/sol_kubernetes_v2/test_prometheus_auto_scaling.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (C) 2022 Fujitsu -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -import ddt -import os -import time - -from tacker.objects import fields -from tacker.tests.functional.sol_kubernetes_v2 import base_v2 -from tacker.tests.functional.sol_kubernetes_v2 import paramgen - - -@ddt.ddt -class PrometheusAutoScalingTest(base_v2.BaseVnfLcmKubernetesV2Test): - - @classmethod - def setUpClass(cls): - super(PrometheusAutoScalingTest, cls).setUpClass() - - cur_dir = os.path.dirname(__file__) - - test_instantiate_cnf_resources_path = os.path.join( - cur_dir, "samples/test_instantiate_cnf_resources") - cls.vnf_pkg_1, cls.vnfd_id_1 = cls.create_vnf_package( - test_instantiate_cnf_resources_path) - - @classmethod - def tearDownClass(cls): - super(PrometheusAutoScalingTest, cls).tearDownClass() - - cls.delete_vnf_package(cls.vnf_pkg_1) - - def setUp(self): - super(PrometheusAutoScalingTest, self).setUp() - - def test_prometheus_auto_scaling_basic(self): - """Test Prometheus Auto Scaling operations with all attributes set - - * About LCM operations: - This test includes the following operations. - - 1. Create a new VNF instance resource - - 2. Instantiate a VNF instance - - 3. Prometheus Auto Scaling alert. - - 4. Terminate a VNF instance - - 5. Delete a VNF instance - """ - - # 1. LCM-Create: Create a new VNF instance resource - # NOTE: extensions and vnfConfigurableProperties are omitted - # because they are commented out in etsi_nfv_sol001. - create_req = paramgen.instantiate_cnf_resources_create(self.vnfd_id_1) - resp, body = self.create_vnf_instance(create_req) - self.assertEqual(201, resp.status_code) - inst_id = body['id'] - - # 2. LCM-Instantiate: Instantiate a VNF instance - vim_id = self.get_k8s_vim_id() - instantiate_req = paramgen.min_sample_instantiate(vim_id) - instantiate_req['additionalParams'][ - 'lcm-kubernetes-def-files'] = ['Files/kubernetes/deployment.yaml'] - instantiate_req['vnfConfigurableProperties'] = { - 'isAutoscaleEnabled': True} - resp, body = self.instantiate_vnf_instance(inst_id, instantiate_req) - self.assertEqual(202, resp.status_code) - - lcmocc_id = os.path.basename(resp.headers['Location']) - self.wait_lcmocc_complete(lcmocc_id) - - resp, body = self.show_vnf_instance(inst_id) - self.assertEqual(200, resp.status_code) - - # 3. Send Auto-Healing alert - alert = paramgen.prometheus_auto_scaling_alert(inst_id) - # CNF scale is not integrated yet. use this value for now. - alert['alerts'][0]['labels']['aspect_id'] = 'invalid_id' - resp, body = self.prometheus_auto_scaling_alert(alert) - self.assertEqual(204, resp.status_code) - time.sleep(5) - - # 4. LCM-Terminate: Terminate VNF - terminate_req = paramgen.terminate_vnf_min() - resp, body = self.terminate_vnf_instance(inst_id, terminate_req) - self.assertEqual(202, resp.status_code) - - lcmocc_id = os.path.basename(resp.headers['Location']) - self.wait_lcmocc_complete(lcmocc_id) - - # wait a bit because there is a bit time lag between lcmocc DB - # update and terminate completion. - time.sleep(10) - - # check instantiationState of VNF - resp, body = self.show_vnf_instance(inst_id) - self.assertEqual(200, resp.status_code) - self.assertEqual(fields.VnfInstanceState.NOT_INSTANTIATED, - body['instantiationState']) - - # 5. LCM-Delete: Delete a VNF instance - resp, body = self.delete_vnf_instance(inst_id) - self.assertEqual(204, resp.status_code) - - # check deletion of VNF instance - resp, body = self.show_vnf_instance(inst_id) - self.assertEqual(404, resp.status_code) diff --git a/tacker/tests/functional/sol_v2/test_prom_auto_scale_heal.py b/tacker/tests/functional/sol_v2/test_prom_auto_scale_heal.py new file mode 100644 index 000000000..607083ef1 --- /dev/null +++ b/tacker/tests/functional/sol_v2/test_prom_auto_scale_heal.py @@ -0,0 +1,361 @@ +# Copyright (C) 2023 Fujitsu +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import ddt +import os +import time + +from tacker.tests.functional.sol_v2_common import paramgen +from tacker.tests.functional.sol_v2_common import test_vnflcm_basic_common + +# Waiting time to trigger autoheal (unit: second) +WAIT_AUTO_HEAL_TIME = 23 + +# Waiting time to lcmocc update in DB (unit: second) +WAIT_LCMOCC_UPDATE_TIME = 3 + + +@ddt.ddt +class PromAutoScaleHealTest(test_vnflcm_basic_common.CommonVnfLcmTest): + + @classmethod + def setUpClass(cls): + super(PromAutoScaleHealTest, cls).setUpClass() + cur_dir = os.path.dirname(__file__) + + # for basic lcms tests min pattern + basic_lcms_min_path = os.path.join( + cur_dir, "../sol_v2_common/samples/basic_lcms_min") + # no image contained + cls.vnf_pkg_2, cls.vnfd_id_2 = cls.create_vnf_package( + basic_lcms_min_path) + + @classmethod + def tearDownClass(cls): + super(PromAutoScaleHealTest, cls).tearDownClass() + cls.delete_vnf_package(cls.vnf_pkg_2) + + def setUp(self): + super(PromAutoScaleHealTest, self).setUp() + + def test_vnfm_auto_heal_vnf(self): + """Test Prometheus VNFM Auto Healing operations + + * About LCM operations: + This test includes the following operations. + - 1. Create a new VNF instance resource + - 2. Instantiate a VNF instance + - 3. Show OpOcc + - 4-5. Receive Alert and Auto Heal + - 6. Show OpOcc + - 7. Receive Alert + - 8-9. Receive Alert and Auto Heal + - 10. Show OpOcc + - 11. Terminate a VNF instance + - 12. Show OpOcc + - 13. Delete a VNF instance + """ + + # 1. Create VNF instance + expected_inst_attrs = [ + 'id', 'vnfdId', 'vnfProductName', 'vnfSoftwareVersion', + 'vnfdVersion', 'instantiationState', '_links' + ] + create_req = paramgen.create_vnf_min(self.vnfd_id_2) + resp, body = self.create_vnf_instance(create_req) + self.assertEqual(201, resp.status_code) + self.check_resp_headers_in_create(resp) + self.check_resp_body(body, expected_inst_attrs) + inst_id = body['id'] + + # 2. Instantiate VNF + instantiate_req = paramgen.instantiate_vnf_min() + instantiate_req['vnfConfigurableProperties'] = { + 'isAutohealEnabled': True} + resp, body = self.instantiate_vnf_instance(inst_id, instantiate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # 3. Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('INSTANTIATE', body['operation']) + + # 4-5. Send alert and auto heal + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + vnfc_info_id = (affected_vnfcs[0]['vduId'] + '-' + + affected_vnfcs[0]['id']) + alert = paramgen.prometheus_auto_healing_alert(inst_id, vnfc_info_id) + resp, body = self.prometheus_auto_healing_alert(alert) + self.assertEqual(204, resp.status_code) + + # Since auto heal takes 20 seconds to trigger, + # wait 23 seconds here. + time.sleep(WAIT_AUTO_HEAL_TIME) + + # 6. Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + heal_lcmocc = [ + heal_lcmocc for heal_lcmocc in body + if heal_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = heal_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('HEAL', body['operation']) + self.assertEqual(2, len(affected_vnfcs)) + + added_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'ADDED'] + self.assertEqual(1, len(added_vnfcs)) + + removed_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'REMOVED'] + self.assertEqual(1, len(removed_vnfcs)) + + removed_vnfc_info_id = (affected_vnfcs[0]['vduId'] + '-' + + affected_vnfcs[0]['id']) + self.assertEqual(vnfc_info_id, removed_vnfc_info_id) + + # 7. Send alert + resp, body = self.show_vnf_instance(inst_id) + self.assertEqual(200, resp.status_code) + + vnfc_infos = body['instantiatedVnfInfo']['vnfcInfo'] + vnfc_info_id_1 = vnfc_infos[0]['id'] + alert = paramgen.prometheus_auto_healing_alert( + inst_id, vnfc_info_id_1) + resp, body = self.prometheus_auto_healing_alert(alert) + self.assertEqual(204, resp.status_code) + + # 8-9. Send alert and auto heal + vnfc_info_id_2 = vnfc_infos[1]['id'] + alert = paramgen.prometheus_auto_healing_alert( + inst_id, vnfc_info_id_2) + resp, body = self.prometheus_auto_healing_alert(alert) + self.assertEqual(204, resp.status_code) + + # Since auto heal takes 20 seconds to trigger, + # wait 23 seconds here. + time.sleep(WAIT_AUTO_HEAL_TIME) + + # 10. Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + heal_lcmocc = [ + heal_lcmocc for heal_lcmocc in body + if heal_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = heal_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('HEAL', body['operation']) + self.assertEqual(4, len(affected_vnfcs)) + + added_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'ADDED'] + self.assertEqual(2, len(added_vnfcs)) + + removed_vnfcs = [ + vnfc for vnfc in affected_vnfcs + if vnfc['changeType'] == 'REMOVED'] + self.assertEqual(2, len(removed_vnfcs)) + + removed_vnfc_info_ids = [ + removed_vnfcs[0]['vduId'] + '-' + removed_vnfcs[0]['id'], + removed_vnfcs[1]['vduId'] + '-' + removed_vnfcs[1]['id'] + ] + self.assertCountEqual( + [vnfc_info_id_1, vnfc_info_id_2], removed_vnfc_info_ids) + + # 11. Terminate a VNF instance + terminate_req = paramgen.terminate_vnf_min() + resp, body = self.terminate_vnf_instance(inst_id, terminate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and terminate completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 12. Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('TERMINATE', body['operation']) + + # 13. Delete a VNF instance + resp, body = self.delete_vnf_instance(inst_id) + self.assertEqual(204, resp.status_code) + + # check deletion of VNF instance + resp, body = self.show_vnf_instance(inst_id) + self.assertEqual(404, resp.status_code) + + def test_vnfm_auto_scale_vnf(self): + """Test Prometheus VNFM Auto Scaling operations + + * About LCM operations: + This test includes the following operations. + - 1. Create a new VNF instance resource + - 2. Instantiate a VNF instance + - 3. Show OpOcc + - 4-5. Receive Alert and Auto Scale out + - 6. Show OpOcc + - 7-8. Receive Alert and Auto Scale in + - 9. Show OpOcc + - 10. Terminate a VNF instance + - 11. Show OpOcc + - 12. Delete a VNF instance + """ + + # 1. Create VNF instance + expected_inst_attrs = [ + 'id', 'vnfdId', 'vnfProductName', 'vnfSoftwareVersion', + 'vnfdVersion', 'instantiationState', '_links' + ] + create_req = paramgen.create_vnf_min(self.vnfd_id_2) + resp, body = self.create_vnf_instance(create_req) + self.assertEqual(201, resp.status_code) + self.check_resp_headers_in_create(resp) + self.check_resp_body(body, expected_inst_attrs) + inst_id = body['id'] + + # 2. Instantiate VNF + instantiate_req = paramgen.instantiate_vnf_min() + instantiate_req['vnfConfigurableProperties'] = { + 'isAutoscaleEnabled': True} + resp, body = self.instantiate_vnf_instance(inst_id, instantiate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # 3. Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('INSTANTIATE', body['operation']) + + # 4-5. Send alert and auto heal + alert = paramgen.prometheus_auto_scaling_alert(inst_id) + resp, body = self.prometheus_auto_scaling_alert(alert) + self.assertEqual(204, resp.status_code) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and scale completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 6. Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + scale_lcmocc = [ + scale_lcmocc for scale_lcmocc in body + if scale_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = scale_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('SCALE', body['operation']) + self.assertEqual(1, len(affected_vnfcs)) + self.assertEqual('ADDED', affected_vnfcs[0]['changeType']) + self.assertEqual( + alert['alerts'][0]['labels']['aspect_id'], + body['operationParams']['aspectId']) + + # 7-8. Send alert and auto-scale in + alert['alerts'][0]['labels']['auto_scale_type'] = 'SCALE_IN' + resp, body = self.prometheus_auto_scaling_alert(alert) + self.assertEqual(204, resp.status_code) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and scale completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 9. Show-OpOcc + filter_expr = {'filter': f'(eq,vnfInstanceId,{inst_id})'} + resp, body = self.list_lcmocc(filter_expr) + self.assertEqual(200, resp.status_code) + + scale_lcmocc = [ + scale_lcmocc for scale_lcmocc in body + if scale_lcmocc['startTime'] == max( + [lcmocc['startTime'] for lcmocc in body])][0] + lcmocc_id = scale_lcmocc['id'] + self.wait_lcmocc_complete(lcmocc_id) + + resp, body = self.show_lcmocc(lcmocc_id) + affected_vnfcs = body['resourceChanges']['affectedVnfcs'] + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('SCALE', body['operation']) + self.assertEqual(1, len(affected_vnfcs)) + self.assertEqual('REMOVED', affected_vnfcs[0]['changeType']) + self.assertEqual( + alert['alerts'][0]['labels']['aspect_id'], + body['operationParams']['aspectId']) + + # 10. Terminate a VNF instance + terminate_req = paramgen.terminate_vnf_min() + resp, body = self.terminate_vnf_instance(inst_id, terminate_req) + self.assertEqual(202, resp.status_code) + + lcmocc_id = os.path.basename(resp.headers['Location']) + self.wait_lcmocc_complete(lcmocc_id) + + # wait a bit because there is a bit time lag between lcmocc DB + # update and terminate completion. + time.sleep(WAIT_LCMOCC_UPDATE_TIME) + + # 11. Show OpOcc + resp, body = self.show_lcmocc(lcmocc_id) + self.assertEqual(200, resp.status_code) + self.assertEqual('COMPLETED', body['operationState']) + self.assertEqual('TERMINATE', body['operation']) + + # 12. Delete a VNF instance + resp, body = self.delete_vnf_instance(inst_id) + self.assertEqual(204, resp.status_code) + + # check deletion of VNF instance + resp, body = self.show_vnf_instance(inst_id) + self.assertEqual(404, resp.status_code) diff --git a/tacker/tests/functional/sol_v2_common/base_v2.py b/tacker/tests/functional/sol_v2_common/base_v2.py index c41f57851..33d8d79c2 100644 --- a/tacker/tests/functional/sol_v2_common/base_v2.py +++ b/tacker/tests/functional/sol_v2_common/base_v2.py @@ -495,6 +495,16 @@ class BaseSolV2Test(base.BaseTestCase): return self.tacker_client.do_request( path, "POST", version="2.0.0") + def prometheus_auto_healing_alert(self, req_body): + path = "/alert/auto_healing" + return self.tacker_client.do_request( + path, "POST", body=req_body) + + def prometheus_auto_scaling_alert(self, req_body): + path = "/alert/auto_scaling" + return self.tacker_client.do_request( + path, "POST", body=req_body) + def create_subscription(self, req_body): path = "/vnflcm/v2/subscriptions" return self.tacker_client.do_request( diff --git a/tacker/tests/functional/sol_v2_common/paramgen.py b/tacker/tests/functional/sol_v2_common/paramgen.py index 4602d6873..eac309f01 100644 --- a/tacker/tests/functional/sol_v2_common/paramgen.py +++ b/tacker/tests/functional/sol_v2_common/paramgen.py @@ -1198,6 +1198,77 @@ def sample4_terminate(): } +def prometheus_auto_healing_alert(inst_id, vnfc_info_id): + return { + "receiver": "receiver", + "status": "firing", + "alerts": [{ + "status": "firing", + "labels": { + "receiver_type": "tacker", + "function_type": "auto_heal", + "vnf_instance_id": inst_id, + "vnfc_info_id": vnfc_info_id + }, + "annotations": { + }, + "startsAt": "2022-06-21T23:47:36.453Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://controller147:9090/graph?g0.expr=" + "up%7Bjob%3D%22node%22%7D+%3D%3D+0&g0.tab=1", + "fingerprint": "5ef77f1f8a3ecb8d" + }], + "groupLabels": {}, + "commonLabels": { + "alertname": "NodeInstanceDown", + "job": "node" + }, + "commonAnnotations": { + "description": "sample" + }, + "externalURL": "http://controller147:9093", + "version": "4", + "groupKey": "{}:{}", + "truncatedAlerts": 0 + } + + +def prometheus_auto_scaling_alert(inst_id): + return { + "receiver": "receiver", + "status": "firing", + "alerts": [{ + "status": "firing", + "labels": { + "receiver_type": "tacker", + "function_type": "auto_scale", + "vnf_instance_id": inst_id, + "auto_scale_type": "SCALE_OUT", + "aspect_id": "VDU1_scale" + }, + "annotations": { + }, + "startsAt": "2022-06-21T23:47:36.453Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://controller147:9090/graph?g0.expr=" + "up%7Bjob%3D%22node%22%7D+%3D%3D+0&g0.tab=1", + "fingerprint": "5ef77f1f8a3ecb8d" + }], + "groupLabels": {}, + "commonLabels": { + "alertname": "NodeInstanceDown", + "job": "node" + }, + "commonAnnotations": { + "description": "sample" + }, + "externalURL": "http://controller147:9093", + "version": "4", + "groupKey": "{}:{}", + "truncatedAlerts": 0 + } + + def server_notification(alarm_id): return { 'notification': { diff --git a/tacker/tests/unit/sol_refactored/common/test_prometheus_plugin.py b/tacker/tests/unit/sol_refactored/common/test_prometheus_plugin.py index a13708334..2e0fc1871 100644 --- a/tacker/tests/unit/sol_refactored/common/test_prometheus_plugin.py +++ b/tacker/tests/unit/sol_refactored/common/test_prometheus_plugin.py @@ -707,6 +707,40 @@ class TestPrometheusPluginFm(base.TestCase): pp._alert, self.request) +class TestPrometheusPluginAutoHealing(base.TestCase): + def setUp(self): + super(TestPrometheusPluginAutoHealing, self).setUp() + objects.register_all() + self.context = context.get_admin_context() + self.request = mock.Mock() + self.request.context = self.context + prometheus_plugin.PrometheusPluginAutoHealing._instance = None + + def tearDown(self): + super(TestPrometheusPluginAutoHealing, self).tearDown() + # delete singleton object + prometheus_plugin.PrometheusPluginAutoHealing._instance = None + + def test_constructor_error(self): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=False) + mon_base.MonitoringPlugin.get_instance( + prometheus_plugin.PrometheusPluginAutoHealing) + self.assertRaises( + SystemError, + prometheus_plugin.PrometheusPluginAutoHealing) + + def test_constructor_stub(self): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=False) + pp = mon_base.MonitoringPlugin.get_instance( + prometheus_plugin.PrometheusPluginAutoHealing) + self.assertIsInstance(pp._instance, mon_base.MonitoringPluginStub) + pp = mon_base.MonitoringPlugin.get_instance( + prometheus_plugin.PrometheusPluginAutoHealing) + self.assertIsInstance(pp._instance, mon_base.MonitoringPluginStub) + + class TestPrometheusPluginAutoScaling(base.TestCase): def setUp(self): super(TestPrometheusPluginAutoScaling, self).setUp() diff --git a/tacker/tests/unit/sol_refactored/conductor/test_prometheus_plugin.py b/tacker/tests/unit/sol_refactored/conductor/test_prometheus_plugin.py index b2d8dfc04..615bdbf1a 100644 --- a/tacker/tests/unit/sol_refactored/conductor/test_prometheus_plugin.py +++ b/tacker/tests/unit/sol_refactored/conductor/test_prometheus_plugin.py @@ -13,10 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. -import webob +import time from tacker import context -from tacker.sol_refactored.common import http_client +from tacker.sol_refactored.common import vnflcm_utils from tacker.sol_refactored.conductor import conductor_v2 from tacker.sol_refactored.conductor import prometheus_plugin_driver as pp_drv from tacker.sol_refactored import objects @@ -119,6 +119,7 @@ class TestPrometheusPlugin(db_base.SqlTestCase): self.context = context.get_admin_context() self.request = mock.Mock() self.request.context = self.context + self.timer_test = (None, None) self.config_fixture.config( group='prometheus_plugin', performance_management=True) self.conductor = conductor_v2.ConductorV2() @@ -129,16 +130,13 @@ class TestPrometheusPlugin(db_base.SqlTestCase): # delete singleton object pp_drv.PrometheusPluginDriver._instance = None - @mock.patch.object(http_client.HttpClient, 'do_request') - def test_request_scale(self, mock_do_request): - resp = webob.Response() - resp.status_code = 202 - mock_do_request.return_value = resp, {} + @mock.patch.object(vnflcm_utils, 'scale') + def test_trigger_scale(self, mock_do_scale): scale_req = { 'type': 'SCALE_OUT', - 'aspect_id': 'vdu', + 'aspectId': 'vdu', } - self.conductor.request_scale( + self.conductor.trigger_scale( self.context, 'vnf_instance_id', scale_req) def test_constructor(self): @@ -150,8 +148,7 @@ class TestPrometheusPlugin(db_base.SqlTestCase): group='prometheus_plugin', performance_management=False) pp_drv.PrometheusPluginDriver._instance = None drv = pp_drv.PrometheusPluginDriver.instance() - drv = pp_drv.PrometheusPluginDriver.instance() - drv.request_scale(None, None, None) + drv.trigger_scale(None, None, None) self.config_fixture.config( group='prometheus_plugin', performance_management=True) drv = pp_drv.PrometheusPluginDriver.instance() @@ -163,3 +160,73 @@ class TestPrometheusPlugin(db_base.SqlTestCase): self.assertRaises( SystemError, pp_drv.PrometheusPluginDriver) + + def test_conductor_vnfm_auto_heal_queue(self): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=True) + pp_drv.PrometheusPluginDriver._instance = None + self.conductor.prom_driver = pp_drv.PrometheusPluginDriver.instance() + self.config_fixture.config( + group='prometheus_plugin', timer_interval=1) + # queueing test + id = 'test_id' + self.conductor.enqueue_auto_heal_instance( + self.context, id, 'id') + self.conductor.enqueue_auto_heal_instance( + self.context, id, 'id2') + self.assertEqual( + self.conductor.prom_driver.timer_map[id].queue, + {'id', 'id2'}) + # Since the timeout period of `VnfmAutoHealTimer` is set to 1 second, + # it is also necessary to wait for 1 second before asserting. + time.sleep(1) + # remove_timer test + self.conductor.dequeue_auto_heal_instance(self.context, id) + self.assertNotIn(id, self.conductor.prom_driver.timer_map) + # remove_timer test: invalid_id + self.conductor.dequeue_auto_heal_instance( + self.context, 'invalid_id') + + @mock.patch.object(vnflcm_utils, 'heal') + def test_conductor_timer_expired(self, mock_do_heal): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=True) + pp_drv.PrometheusPluginDriver._instance = None + self.conductor.prom_driver = pp_drv.PrometheusPluginDriver.instance() + self.conductor.prom_driver._timer_expired( + self.context, 'test_id', ['id']) + + def expired(self, context, id, queue): + queue.sort() + self.timer_test = (id, queue) + + def test_timer(self): + # queueing test + timer = pp_drv.VnfmAutoHealTimer(self.context, 'id', 1, self.expired) + timer.add_vnfc_info_id('1') + timer.add_vnfc_info_id('3') + # Since the timeout period of `VnfmAutoHealTimer` is set to 1 second, + # it is also necessary to wait for 1 second before asserting. + time.sleep(1) + self.assertEqual(self.timer_test[0], 'id') + self.assertEqual(self.timer_test[1], ['1', '3']) + + def test_timer_cancel(self): + # cancel test + timer = pp_drv.VnfmAutoHealTimer(self.context, 'id2', 1, self.expired) + timer.add_vnfc_info_id('5') + timer.cancel() + # Since the timeout period of `VnfmAutoHealTimer` is set to 1 second, + # it is also necessary to wait for 1 second before asserting. + time.sleep(1) + self.assertIsNone(self.timer_test[0]) + self.assertIsNone(self.timer_test[1]) + + def test_timer_destructor(self): + # method call after cancel() + timer = pp_drv.VnfmAutoHealTimer(self.context, 'id', 1, self.expired) + timer.cancel() + timer.expire() + timer.add_vnfc_info_id(['4']) + timer.cancel() + timer.__del__() diff --git a/tacker/tests/unit/sol_refactored/controller/test_prometheus_plugin.py b/tacker/tests/unit/sol_refactored/controller/test_prometheus_plugin.py index 505e1375a..b2d082973 100644 --- a/tacker/tests/unit/sol_refactored/controller/test_prometheus_plugin.py +++ b/tacker/tests/unit/sol_refactored/controller/test_prometheus_plugin.py @@ -145,15 +145,65 @@ _body_scale_alert1 = { 'fingerprint': '5ef77f1f8a3ecb8d' } +_body_heal_alert1 = { + 'status': 'firing', + 'labels': { + 'receiver_type': 'tacker', + 'function_type': 'auto_heal', + 'vnf_instance_id': 'vnf instance id', + 'vnfc_info_id': 'vnfc info id' + }, + 'annotations': { + }, + 'startsAt': '2022-06-21T23:47:36.453Z', + 'endsAt': '0001-01-01T00:00:00Z', + 'generatorURL': 'http://controller147:9090/graph?g0.expr=' + 'up%7Bjob%3D%22node%22%7D+%3D%3D+0&g0.tab=1', + 'fingerprint': '5ef77f1f8a3ecb8d' +} + # function_type mismatch _body_scale_alert2 = copy.deepcopy(_body_scale_alert1) _body_scale_alert2['labels']['function_type'] = 'vnffm' +_body_scale_alert3 = copy.deepcopy(_body_scale_alert1) +_body_scale_alert3['status'] = 'resolved' + +_body_scale_alert4 = copy.deepcopy(_body_scale_alert1) +_body_scale_alert4['labels']['function_type'] = 'auto_heal' + +_body_scale_alert5 = copy.deepcopy(_body_scale_alert1) +_body_scale_alert5['labels']['aspect_id'] = 'aspect id' + _body_scale = copy.deepcopy(_body_base) _body_scale.update({ 'alerts': [_body_scale_alert1, _body_scale_alert2] }) +_body_scale_continue = copy.deepcopy(_body_base) +_body_scale_continue.update({ + 'alerts': [_body_scale_alert3, _body_scale_alert4, _body_scale_alert5] +}) + +_body_heal = copy.deepcopy(_body_base) +_body_heal.update({ + 'alerts': [_body_heal_alert1] +}) + +_body_heal_alert2 = copy.deepcopy(_body_heal_alert1) +_body_heal_alert2['status'] = 'resolved' + +_body_heal_alert3 = copy.deepcopy(_body_heal_alert1) +_body_heal_alert3['labels']['function_type'] = 'auto_scale' + +_body_heal_alert4 = copy.deepcopy(_body_heal_alert1) +_body_heal_alert4['labels']['vnfc_info_id'] = 'vnfcInfoId' + +_body_heal_continue = copy.deepcopy(_body_base) +_body_heal_continue.update({ + 'alerts': [_body_heal_alert2, _body_heal_alert3, _body_heal_alert4] +}) + _inst1 = { 'id': 'test_id', 'vnfdId': 'vnfdId', @@ -187,6 +237,14 @@ _inst1 = { 'vduId': 'vdu_id', 'vnfcResourceInfoId': 'id2', 'vnfcState': 'STARTED' + }, { + 'id': 'vnfc info id', + 'vduId': 'vdu_id', + 'vnfcResourceInfoId': 'id2', + 'vnfcState': 'STARTED' + }], + 'scaleStatus': [{ + 'aspectId': 'aspect' }] }, 'metadata': { @@ -198,6 +256,31 @@ _inst2.update({ 'vnfConfigurableProperties': { 'isAutoscaleEnabled': True }, + 'instantiationState': 'INSTANTIATED' +}) + +_inst3 = copy.deepcopy(_inst1) +_inst3.update({ + 'vnfConfigurableProperties': { + 'isAutoscaleEnabled': False + }, + 'instantiationState': 'INSTANTIATED' +}) + +_inst4 = copy.deepcopy(_inst1) +_inst4.update({ + 'vnfConfigurableProperties': { + 'isAutohealEnabled': False + }, + 'instantiationState': 'INSTANTIATED' +}) + +_inst5 = copy.deepcopy(_inst1) +_inst5.update({ + 'vnfConfigurableProperties': { + 'isAutohealEnabled': True + }, + 'instantiationState': 'INSTANTIATED' }) datetime_test = datetime.datetime.fromisoformat( @@ -324,6 +407,64 @@ class TestPrometheusPluginFm(base.TestCase): self.assertEqual(204, result.status) +class TestPrometheusPluginAutoHealing(base.TestCase): + def setUp(self): + super(TestPrometheusPluginAutoHealing, self).setUp() + objects.register_all() + self.context = context.get_admin_context() + self.request = mock.Mock() + self.request.context = self.context + self.controller = prometheus_plugin_controller.AutoHealingController() + plugin.PrometheusPluginAutoHealing._instance = None + + def tearDown(self): + super(TestPrometheusPluginAutoHealing, self).tearDown() + # delete singleton object + plugin.PrometheusPluginAutoHealing._instance = None + + def test_auto_healing_config_false(self): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=False) + self.assertRaises( + sol_ex.PrometheusPluginNotEnabled, + self.controller.auto_healing, self.request, {}) + + @mock.patch.object(inst_utils, 'get_inst') + def test_auto_healing_no_autoheal_enabled(self, mock_inst): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=True) + mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst4) + result = self.controller.auto_healing( + self.request, _body_heal) + self.assertEqual(204, result.status) + + @mock.patch.object(inst_utils, 'get_inst') + def test_auto_healing_is_autoheal_enabled(self, mock_inst): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=True) + mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst5) + result = self.controller.auto_healing(self.request, _body_heal) + self.assertEqual(204, result.status) + + @mock.patch.object(inst_utils, 'get_inst') + def test_auto_healing_multiple_continue(self, mock_inst): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=True) + mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst5) + result = self.controller.auto_healing( + self.request, _body_heal_continue) + self.assertEqual(204, result.status) + + @mock.patch.object(inst_utils, 'get_inst') + def test_auto_healing_not_instantiated(self, mock_inst): + self.config_fixture.config( + group='prometheus_plugin', auto_healing=True) + mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst1) + result = self.controller.auto_healing( + self.request, _body_heal) + self.assertEqual(204, result.status) + + class TestPrometheusPluginAutoScaling(base.TestCase): def setUp(self): super(TestPrometheusPluginAutoScaling, self).setUp() @@ -344,15 +485,15 @@ class TestPrometheusPluginAutoScaling(base.TestCase): group='prometheus_plugin', auto_scaling=False) self.assertRaises( sol_ex.PrometheusPluginNotEnabled, - self.controller.auto_scaling_id, self.request, 'id', {}) + self.controller.auto_scaling, self.request, {}) @mock.patch.object(inst_utils, 'get_inst') def test_auto_scaling_no_autoscale_enabled(self, mock_inst): self.config_fixture.config( group='prometheus_plugin', auto_scaling=True) - mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst1) - result = self.controller.auto_scaling_id( - self.request, 'id', _body_scale) + mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst3) + result = self.controller.auto_scaling( + self.request, _body_scale) self.assertEqual(204, result.status) @mock.patch.object(inst_utils, 'get_inst') @@ -378,3 +519,21 @@ class TestPrometheusPluginAutoScaling(base.TestCase): group='prometheus_plugin', auto_scaling=True) result = self.controller.auto_scaling(self.request, {}) self.assertEqual(204, result.status) + + @mock.patch.object(inst_utils, 'get_inst') + def test_auto_scaling_multiple_continue(self, mock_inst): + self.config_fixture.config( + group='prometheus_plugin', auto_scaling=True) + mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst2) + result = self.controller.auto_scaling( + self.request, _body_scale_continue) + self.assertEqual(204, result.status) + + @mock.patch.object(inst_utils, 'get_inst') + def test_auto_scaling_not_instantiated(self, mock_inst): + self.config_fixture.config( + group='prometheus_plugin', auto_scaling=True) + mock_inst.return_value = objects.VnfInstanceV2.from_dict(_inst1) + result = self.controller.auto_scaling( + self.request, _body_scale) + self.assertEqual(204, result.status)