diff --git a/.zuul.yaml b/.zuul.yaml index 887de3166..977fe51a4 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -67,6 +67,7 @@ - openstack/python-tackerclient - openstack/tacker - openstack/tacker-horizon + - x/fenix vars: devstack_localrc: CELLSV2_SETUP: singleconductor @@ -93,6 +94,7 @@ mistral: https://opendev.org/openstack/mistral tacker: https://opendev.org/openstack/tacker blazar: https://opendev.org/openstack/blazar + fenix: https://opendev.org/x/fenix devstack_services: # Core services enabled for this branch. # This list replaces the test-matrix. diff --git a/devstack/lib/tacker b/devstack/lib/tacker index b34d767c6..b5c08a774 100644 --- a/devstack/lib/tacker +++ b/devstack/lib/tacker @@ -81,6 +81,7 @@ TACKER_NOVA_CA_CERTIFICATES_FILE=${TACKER_NOVA_CA_CERTIFICATES_FILE:-} TACKER_NOVA_API_INSECURE=${TACKER_NOVA_API_INSECURE:-False} HEAT_CONF_DIR=/etc/heat +CEILOMETER_CONF_DIR=/etc/ceilometer source ${TACKER_DIR}/tacker/tests/contrib/post_test_hook_lib.sh @@ -480,3 +481,11 @@ function modify_heat_flavor_policy_rule { # Allow non-admin projects with 'admin' roles to create flavors in Heat echo '"resource_types:OS::Nova::Flavor": "role:admin"' >> $policy_file } + +function configure_maintenance_event_types { + local event_definitions_file=$CEILOMETER_CONF_DIR/event_definitions.yaml + local maintenance_events_file=$TACKER_DIR/etc/ceilometer/maintenance_event_types.yaml + + echo "Configure maintenance event types to $event_definitions_file" + cat $maintenance_events_file >> $event_definitions_file +} diff --git a/devstack/local.conf.example b/devstack/local.conf.example index 62ddaf2b3..7655cd904 100644 --- a/devstack/local.conf.example +++ b/devstack/local.conf.example @@ -43,12 +43,16 @@ enable_plugin mistral https://opendev.org/openstack/mistral master # Ceilometer #CEILOMETER_PIPELINE_INTERVAL=300 +CEILOMETER_EVENT_ALARM=True enable_plugin ceilometer https://opendev.org/openstack/ceilometer master enable_plugin aodh https://opendev.org/openstack/aodh master # Blazar enable_plugin blazar https://github.com/openstack/blazar.git master +# Fenix +enable_plugin fenix https://opendev.org/x/fenix.git master + # Tacker enable_plugin tacker https://opendev.org/openstack/tacker master diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 9669bddd4..1d050d6ce 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -41,6 +41,11 @@ if is_service_enabled tacker; then tacker_check_and_download_images echo_summary "Registering default VIM" tacker_register_default_vim + + if is_service_enabled ceilometer; then + echo_summary "Configure maintenance event types" + configure_maintenance_event_types + fi fi fi diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst index 8b2e5c3da..fe78a3d49 100644 --- a/doc/source/reference/index.rst +++ b/doc/source/reference/index.rst @@ -24,3 +24,4 @@ Reference mistral_workflows_usage_guide.rst block_storage_usage_guide.rst reservation_policy_usage_guide.rst + maintenance_usage_guide.rst diff --git a/doc/source/reference/maintenance_usage_guide.rst b/doc/source/reference/maintenance_usage_guide.rst new file mode 100644 index 000000000..926a194d4 --- /dev/null +++ b/doc/source/reference/maintenance_usage_guide.rst @@ -0,0 +1,183 @@ +.. + Copyright 2020 Distributed Cloud and Network (DCN) + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +================================ +VNF zero impact host maintenance +================================ + +Tacker allows you to maintenance host with VNF zero impact. Maintenance +workflows will be performed in the ``Fenix`` service by creating a session +which can do scaling, migrating VNFs and patch hosts. + + +References +~~~~~~~~~~ + +- `Fenix `_. +- `Fenix Configuration Guide `_. + +Installation and configurations +------------------------------- + +1. You need Fenix, Ceilometer and Aodh OpenStack services. + +2. Modify the below configuration files: + +/etc/ceilometer/event_pipeline.yaml + +.. code-block:: yaml + + sinks: + - name: event_sink + publishers: + - panko:// + - notifier:// + - notifier://?topic=alarm.all + +/etc/ceilometer/event_definitions.yaml: + +.. code-block:: yaml + + - event_type: 'maintenance.scheduled' + traits: + service: + fields: payload.service + allowed_actions: + fields: payload.allowed_actions + instance_ids: + fields: payload.instance_ids + reply_url: + fields: payload.reply_url + state: + fields: payload.state + session_id: + fields: payload.session_id + actions_at: + fields: payload.actions_at + type: datetime + project_id: + fields: payload.project_id + reply_at: + fields: payload.reply_at + type: datetime + metadata: + fields: payload.metadata + - event_type: 'maintenance.host' + traits: + host: + fields: payload.host + project_id: + fields: payload.project_id + session_id: + fields: payload.session_id + state: + fields: payload.state + + +Deploying maintenance tosca template with tacker +------------------------------------------------ + +When template is normal +~~~~~~~~~~~~~~~~~~~~~~~ + +If ``Fenix`` service is enabled and maintenance event_types are defined, then +all VNF created by legacy VNFM will get ``ALL_MAINTENANCE`` resource in Stack. + +.. code-block:: yaml + + resources: + ALL_maintenance: + properties: + alarm_actions: + - http://openstack-master:9890/v1.0/vnfs/e8b9bec5-541b-492c-954e-cd4af71eda1f/maintenance/0cc65f4bba9c42bfadf4aebec6ae7348/hbyhgkav + event_type: maintenance.scheduled + type: OS::Aodh::EventAlarm + +When template has maintenance property +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If VDU in VNFD has maintenance property, then VNFM creates +``[VDU_NAME]_MAINTENANCE`` alarm resources and will be use for VNF software +modification later. This is not works yet. It will be updated. + +``Sample tosca-template``: + +.. code-block:: yaml + + tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0 + + description: VNF TOSCA template with maintenance + + metadata: + template_name: sample-tosca-vnfd-maintenance + + topology_template: + node_templates: + VDU1: + type: tosca.nodes.nfv.VDU.Tacker + properties: + maintenance: True + image: cirros-0.4.0-x86_64-disk + capabilities: + nfv_compute: + properties: + disk_size: 1 GB + mem_size: 512 MB + num_cpus: 2 + + CP1: + type: tosca.nodes.nfv.CP.Tacker + properties: + management: true + order: 0 + anti_spoofing_protection: false + requirements: + - virtualLink: + node: VL1 + - virtualBinding: + node: VDU1 + + VL1: + type: tosca.nodes.nfv.VL + properties: + network_name: net_mgmt + vendor: Tacker + + policies: + - SP1: + type: tosca.policies.tacker.Scaling + properties: + increment: 1 + cooldown: 120 + min_instances: 1 + max_instances: 3 + default_instances: 2 + targets: [VDU1] + + +Configure maintenance constraints with config yaml +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When ``Fenix`` does maintenance, it requires some constraints for zero impact. +Like below config file, each VNF can set and update constraints. + +.. code-block:: yaml + + maintenance: + max_impacted_members: 1 + recovery_time: 60, + mitigation_type: True, + lead_time: 120, + migration_type: 'MIGRATE' diff --git a/etc/ceilometer/maintenance_event_types.yaml b/etc/ceilometer/maintenance_event_types.yaml new file mode 100644 index 000000000..c60cc97ad --- /dev/null +++ b/etc/ceilometer/maintenance_event_types.yaml @@ -0,0 +1,34 @@ +- event_type: 'maintenance.scheduled' + traits: + service: + fields: payload.service + allowed_actions: + fields: payload.allowed_actions + instance_ids: + fields: payload.instance_ids + reply_url: + fields: payload.reply_url + state: + fields: payload.state + session_id: + fields: payload.session_id + actions_at: + fields: payload.actions_at + type: datetime + project_id: + fields: payload.project_id + reply_at: + fields: payload.reply_at + type: datetime + metadata: + fields: payload.metadata +- event_type: 'maintenance.host' + traits: + host: + fields: payload.host + project_id: + fields: payload.project_id + session_id: + fields: payload.session_id + state: + fields: payload.state diff --git a/setup.cfg b/setup.cfg index 515961a0d..136381d05 100644 --- a/setup.cfg +++ b/setup.cfg @@ -93,6 +93,8 @@ oslo.config.opts = tacker.vnfm.monitor_drivers.ceilometer.ceilometer = tacker.vnfm.monitor_drivers.ceilometer.ceilometer:config_opts tacker.vnfm.monitor_drivers.zabbix.zabbix = tacker.vnfm.monitor_drivers.zabbix.zabbix:config_opts tacker.alarm_receiver = tacker.alarm_receiver:config_opts + tacker.plugins.fenix = tacker.plugins.fenix:config_opts + mistral.actions = tacker.vim_ping_action = tacker.nfvo.workflows.vim_monitor.vim_ping_action:PingVimAction diff --git a/tacker/alarm_receiver.py b/tacker/alarm_receiver.py index 20e9f6d73..514dfeec9 100644 --- a/tacker/alarm_receiver.py +++ b/tacker/alarm_receiver.py @@ -52,6 +52,8 @@ class AlarmReceiver(wsgi.Middleware): if not self.handle_url(url): return prefix, info, params = self.handle_url(req.url) + resource = 'trigger' if info[4] != 'maintenance' else 'maintenance' + redirect = resource + 's' auth = cfg.CONF.keystone_authtoken alarm_auth = cfg.CONF.alarm_auth token = Token(username=alarm_auth.username, @@ -66,19 +68,24 @@ class AlarmReceiver(wsgi.Middleware): # Change the body request if req.body: body_dict = dict() - body_dict['trigger'] = {} - body_dict['trigger'].setdefault('params', {}) + body_dict[resource] = {} + body_dict[resource].setdefault('params', {}) # Update params in the body request body_info = jsonutils.loads(req.body) - body_dict['trigger']['params']['data'] = body_info - body_dict['trigger']['params']['credential'] = info[6] - # Update policy and action - body_dict['trigger']['policy_name'] = info[4] - body_dict['trigger']['action_name'] = info[5] + body_dict[resource]['params']['credential'] = info[6] + if resource == 'maintenance': + body_info.update({ + 'body': self._handle_maintenance_body(body_info)}) + del body_info['reason_data'] + else: + # Update policy and action + body_dict[resource]['policy_name'] = info[4] + body_dict[resource]['action_name'] = info[5] + body_dict[resource]['params']['data'] = body_info req.body = jsonutils.dump_as_bytes(body_dict) LOG.debug('Body alarm: %s', req.body) # Need to change url because of mandatory - req.environ['PATH_INFO'] = prefix + 'triggers' + req.environ['PATH_INFO'] = prefix + redirect req.environ['QUERY_STRING'] = '' LOG.debug('alarm url in receiver: %s', req.url) @@ -98,3 +105,15 @@ class AlarmReceiver(wsgi.Middleware): prefix_url = '/%(collec)s/%(vnf_uuid)s/' % {'collec': p[2], 'vnf_uuid': p[3]} return prefix_url, p, params + + def _handle_maintenance_body(self, body_info): + body = {} + traits_list = body_info['reason_data']['event']['traits'] + if type(traits_list) is not list: + return + for key, t_type, val in traits_list: + if t_type == 1 and val and (val[0] == '[' or val[0] == '{'): + body[key] = eval(val) + else: + body[key] = val + return body diff --git a/tacker/extensions/vnfm.py b/tacker/extensions/vnfm.py index 6231c8695..d9c25a2c7 100644 --- a/tacker/extensions/vnfm.py +++ b/tacker/extensions/vnfm.py @@ -208,6 +208,10 @@ class InvalidInstReqInfoForScaling(exceptions.InvalidInput): "fixed ip_address or mac_address.") +class InvalidMaintenanceParameter(exceptions.InvalidInput): + message = _("Could not find the required params for maintenance") + + def _validate_service_type_list(data, valid_values=None): if not isinstance(data, list): msg = _("Invalid data format for service list: '%s'") % data @@ -491,6 +495,37 @@ SUB_RESOURCE_ATTRIBUTE_MAP = { } } } + }, + 'maintenances': { + 'parent': { + 'collection_name': 'vnfs', + 'member_name': 'vnf' + }, + 'members': { + 'maintenance': { + 'parameters': { + 'params': { + 'allow_post': True, + 'allow_put': False, + 'is_visible': True, + 'validate': {'type:dict_or_none': None} + }, + 'tenant_id': { + 'allow_post': True, + 'allow_put': False, + 'validate': {'type:string': None}, + 'required_by_policy': False, + 'is_visible': False + }, + 'response': { + 'allow_post': False, + 'allow_put': False, + 'validate': {'type:dict_or_none': None}, + 'is_visible': True + } + } + } + } } } @@ -623,3 +658,7 @@ class VNFMPluginBase(service_base.NFVPluginBase): def create_vnf_trigger( self, context, vnf_id, trigger): pass + + @abc.abstractmethod + def create_vnf_maintenance(self, context, vnf_id, maintenance): + pass diff --git a/tacker/objects/heal_vnf_request.py b/tacker/objects/heal_vnf_request.py index fb0b064bd..b26c6031a 100644 --- a/tacker/objects/heal_vnf_request.py +++ b/tacker/objects/heal_vnf_request.py @@ -36,11 +36,13 @@ class HealVnfRequest(base.TackerObject): # Version 1.0: Initial version # Version 1.1: Added vnf_instance_id - VERSION = '1.1' + # Version 1.2: Added stack_id for nested heat-template + VERSION = '1.2' fields = { 'vnfc_instance_id': fields.ListOfStringsField(nullable=True, default=[]), + 'stack_id': fields.StringField(nullable=True, default=''), 'cause': fields.StringField(nullable=True, default=None), 'additional_params': fields.ListOfObjectsField( 'HealVnfAdditionalParams', default=[]) diff --git a/tacker/plugins/common/constants.py b/tacker/plugins/common/constants.py index 0945a1743..cee94ea3e 100644 --- a/tacker/plugins/common/constants.py +++ b/tacker/plugins/common/constants.py @@ -30,6 +30,7 @@ COMMON_PREFIXES = { # Service operation status constants ACTIVE = "ACTIVE" +ACK = "ACK" PENDING_CREATE = "PENDING_CREATE" PENDING_UPDATE = "PENDING_UPDATE" @@ -40,6 +41,7 @@ PENDING_HEAL = "PENDING_HEAL" DEAD = "DEAD" ERROR = "ERROR" +NACK = "NACK" ACTIVE_PENDING_STATUSES = ( ACTIVE, @@ -72,6 +74,10 @@ RES_EVT_SCALE = "SCALE" RES_EVT_NA_STATE = "Not Applicable" RES_EVT_ONBOARDED = "OnBoarded" RES_EVT_HEAL = "HEAL" +RES_EVT_MAINTENANCE = [ + "MAINTENANCE", "SCALE_IN", "MAINTENANCE_COMPLETE", + "PREPARE_MAINTENANCE", "PLANNED_MAINTENANCE", "INSTANCE_ACTION_DONE" +] VNF_STATUS_TO_EVT_TYPES = {PENDING_CREATE: RES_EVT_CREATE, diff --git a/tacker/plugins/fenix.py b/tacker/plugins/fenix.py new file mode 100644 index 000000000..835a01255 --- /dev/null +++ b/tacker/plugins/fenix.py @@ -0,0 +1,456 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import requests +import time +import yaml + +from oslo_config import cfg +from oslo_serialization import jsonutils + +from tacker.common import clients +from tacker.common import log +from tacker.extensions import vnfm +from tacker.plugins.common import constants +from tacker.vnfm import vim_client + + +CONF = cfg.CONF +OPTS = [ + cfg.IntOpt('lead_time', default=120, + help=_('Time for migration_type operation')), + cfg.IntOpt('max_interruption_time', default=120, + help=_('Time for how long live migration can take')), + cfg.IntOpt('recovery_time', default=2, + help=_('Time for migrated node could be fully running state')), + cfg.IntOpt('request_retries', + default=5, + help=_("Number of attempts to retry for request")), + cfg.IntOpt('request_retry_wait', + default=5, + help=_("Wait time (in seconds) between consecutive request")) +] +CONF.register_opts(OPTS, 'fenix') +MAINTENANCE_KEYS = ( + 'instance_ids', 'session_id', 'state', 'reply_url' +) +MAINTENANCE_SUB_KEYS = { + 'PREPARE_MAINTENANCE': [('allowed_actions', 'list'), + ('instance_ids', 'list')], + 'PLANNED_MAINTENANCE': [('allowed_actions', 'list'), + ('instance_ids', 'list')] +} + + +def config_opts(): + return [('fenix', OPTS)] + + +class FenixPlugin(object): + def __init__(self): + self.REQUEST_RETRIES = cfg.CONF.fenix.request_retries + self.REQUEST_RETRY_WAIT = cfg.CONF.fenix.request_retry_wait + self.endpoint = None + self._instances = {} + self.vim_client = vim_client.VimClient() + + @log.log + def request(self, plugin, context, vnf_dict, maintenance={}, + data_func=None): + params_list = [maintenance] + method = 'put' + is_reply = True + if data_func: + action, create_func = data_func.split('_', 1) + create_func = '_create_%s_list' % create_func + if action in ['update', 'delete'] and hasattr(self, create_func): + params_list = getattr(self, create_func)( + context, vnf_dict, action) + method = action if action == 'delete' else 'put' + is_reply = False + for params in params_list: + self._request(plugin, context, vnf_dict, params, method, is_reply) + return len(params_list) + + @log.log + def create_vnf_constraints(self, plugin, context, vnf_dict): + self.update_vnf_constraints(plugin, context, vnf_dict, + objects=['instance_group', + 'project_instance']) + + @log.log + def delete_vnf_constraints(self, plugin, context, vnf_dict): + self.update_vnf_constraints(plugin, context, vnf_dict, + action='delete', + objects=['instance_group', + 'project_instance']) + + @log.log + def update_vnf_instances(self, plugin, context, vnf_dict, + action='update'): + requests = self.update_vnf_constraints(plugin, context, + vnf_dict, action, + objects=['project_instance']) + if requests[0]: + self.post(context, vnf_dict) + + @log.log + def update_vnf_constraints(self, plugin, context, vnf_dict, + action='update', objects=[]): + result = [] + for obj in objects: + requests = self.request(plugin, context, vnf_dict, + data_func='%s_%s' % (action, obj)) + result.append(requests) + return result + + @log.log + def post(self, context, vnf_dict, **kwargs): + post_function = getattr(context, 'maintenance_post_function', None) + if not post_function: + return + post_function(context, vnf_dict) + del context.maintenance_post_function + + @log.log + def project_instance_pre(self, context, vnf_dict): + key = vnf_dict['id'] + if key not in self._instances: + self._instances.update({ + key: self._get_instances(context, vnf_dict)}) + + @log.log + def validate_maintenance(self, maintenance): + body = maintenance['maintenance']['params']['data']['body'] + if not set(MAINTENANCE_KEYS).issubset(body) or \ + body['state'] not in constants.RES_EVT_MAINTENANCE: + raise vnfm.InvalidMaintenanceParameter() + sub_keys = MAINTENANCE_SUB_KEYS.get(body['state'], ()) + for key, val_type in sub_keys: + if key not in body or type(body[key]) is not eval(val_type): + raise vnfm.InvalidMaintenanceParameter() + return body + + @log.log + def _request(self, plugin, context, vnf_dict, maintenance, + method='put', is_reply=True): + client = self._get_openstack_clients(context, vnf_dict) + if not self.endpoint: + self.endpoint = client.keystone_session.get_endpoint( + service_type='maintenance', region_name=client.region_name) + if not self.endpoint: + raise vnfm.ServiceTypeNotFound(service_type_id='maintenance') + + if 'reply_url' in maintenance: + url = maintenance['reply_url'] + elif 'url' in maintenance: + url = "%s/%s" % (self.endpoint.rstrip('/'), + maintenance['url'].strip('/')) + else: + return + + def create_headers(): + return { + 'X-Auth-Token': client.keystone_session.get_token(), + 'Content-Type': 'application/json', + 'Accept': 'application/json' + } + + request_body = {} + request_body['headers'] = create_headers() + state = constants.ACK if vnf_dict['status'] == constants.ACTIVE \ + else constants.NACK + if method == 'put': + data = maintenance.get('data', {}) + if is_reply: + data['session_id'] = maintenance.get('session_id', '') + data['state'] = "%s_%s" % (state, maintenance['state']) + request_body['data'] = jsonutils.dump_as_bytes(data) + + def request_wait(): + retries = self.REQUEST_RETRIES + while retries > 0: + response = getattr(requests, method)(url, **request_body) + if response.status_code == 200: + break + else: + retries -= 1 + time.sleep(self.REQUEST_RETRY_WAIT) + + plugin.spawn_n(request_wait) + + @log.log + def handle_maintenance(self, plugin, context, maintenance): + action = '_create_%s' % maintenance['state'].lower() + maintenance['data'] = {} + if hasattr(self, action): + getattr(self, action)(plugin, context, maintenance) + + @log.log + def _create_maintenance(self, plugin, context, maintenance): + vnf_dict = maintenance.get('vnf', {}) + vnf_dict['attributes'].update({'maintenance_scaled': 0}) + plugin._update_vnf_post(context, vnf_dict['id'], constants.ACTIVE, + vnf_dict, constants.ACTIVE, + constants.RES_EVT_UPDATE) + instances = self._get_instances(context, vnf_dict) + instance_ids = [x['id'] for x in instances] + maintenance['data'].update({'instance_ids': instance_ids}) + + @log.log + def _create_scale_in(self, plugin, context, maintenance): + def post_function(context, vnf_dict): + scaled = int(vnf_dict['attributes'].get('maintenance_scaled', 0)) + vnf_dict['attributes']['maintenance_scaled'] = str(scaled + 1) + plugin._update_vnf_post(context, vnf_dict['id'], constants.ACTIVE, + vnf_dict, constants.ACTIVE, + constants.RES_EVT_UPDATE) + instances = self._get_instances(context, vnf_dict) + instance_ids = [x['id'] for x in instances] + maintenance['data'].update({'instance_ids': instance_ids}) + self.request(plugin, context, vnf_dict, maintenance) + + vnf_dict = maintenance.get('vnf', {}) + policy_action = self._create_scale_dict(plugin, context, vnf_dict) + if policy_action: + maintenance.update({'policy_action': policy_action}) + context.maintenance_post_function = post_function + + @log.log + def _create_prepare_maintenance(self, plugin, context, maintenance): + self._create_planned_maintenance(plugin, context, maintenance) + + @log.log + def _create_planned_maintenance(self, plugin, context, maintenance): + def post_function(context, vnf_dict): + migration_type = self._get_constraints(vnf_dict, + key='migration_type', + default='MIGRATE') + maintenance['data'].update({'instance_action': migration_type}) + self.request(plugin, context, vnf_dict, maintenance) + + vnf_dict = maintenance.get('vnf', {}) + instances = self._get_instances(context, vnf_dict) + request_instance_id = maintenance['instance_ids'][0] + selected = None + for instance in instances: + if instance['id'] == request_instance_id: + selected = instance + break + if not selected: + vnfm.InvalidMaintenanceParameter() + + migration_type = self._get_constraints(vnf_dict, key='migration_type', + default='MIGRATE') + if migration_type == 'OWN_ACTION': + policy_action = self._create_migrate_dict(context, vnf_dict, + selected) + maintenance.update({'policy_action': policy_action}) + context.maintenance_post_function = post_function + else: + post_function(context, vnf_dict) + + @log.log + def _create_maintenance_complete(self, plugin, context, maintenance): + def post_function(context, vnf_dict): + vim_res = self.vim_client.get_vim(context, vnf_dict['vim_id']) + scaled = int(vnf_dict['attributes'].get('maintenance_scaled', 0)) + if vim_res['vim_type'] == 'openstack': + scaled -= 1 + vnf_dict['attributes']['maintenance_scaled'] = str(scaled) + plugin._update_vnf_post(context, vnf_dict['id'], + constants.ACTIVE, vnf_dict, + constants.ACTIVE, + constants.RES_EVT_UPDATE) + if scaled > 0: + scale_out(plugin, context, vnf_dict) + else: + instances = self._get_instances(context, vnf_dict) + instance_ids = [x['id'] for x in instances] + maintenance['data'].update({'instance_ids': instance_ids}) + self.request(plugin, context, vnf_dict, maintenance) + + def scale_out(plugin, context, vnf_dict): + policy_action = self._create_scale_dict(plugin, context, vnf_dict, + scale_type='out') + context.maintenance_post_function = post_function + plugin._vnf_action.invoke(policy_action['action'], + 'execute_action', plugin=plugin, + context=context, vnf_dict=vnf_dict, + args=policy_action['args']) + + vnf_dict = maintenance.get('vnf', {}) + scaled = vnf_dict.get('attributes', {}).get('maintenance_scaled', 0) + if int(scaled): + policy_action = self._create_scale_dict(plugin, context, vnf_dict, + scale_type='out') + maintenance.update({'policy_action': policy_action}) + context.maintenance_post_function = post_function + + @log.log + def _create_scale_dict(self, plugin, context, vnf_dict, scale_type='in'): + policy_action, scale_dict = {}, {} + policies = self._get_scaling_policies(plugin, context, vnf_dict) + if not policies: + return + scale_dict['type'] = scale_type + scale_dict['policy'] = policies[0]['name'] + policy_action['action'] = 'autoscaling' + policy_action['args'] = {'scale': scale_dict} + return policy_action + + @log.log + def _create_migrate_dict(self, context, vnf_dict, instance): + policy_action, heal_dict = {}, {} + heal_dict['vdu_name'] = instance['name'] + heal_dict['cause'] = ["Migrate resource '%s' to other host."] + heal_dict['stack_id'] = instance['stack_name'] + if 'scaling_group_names' in vnf_dict['attributes']: + sg_names = vnf_dict['attributes']['scaling_group_names'] + sg_names = list(jsonutils.loads(sg_names).keys()) + heal_dict['heat_tpl'] = '%s_res.yaml' % sg_names[0] + policy_action['action'] = 'vdu_autoheal' + policy_action['args'] = heal_dict + return policy_action + + @log.log + def _create_instance_group_list(self, context, vnf_dict, action): + group_id = vnf_dict['attributes'].get('maintenance_group', '') + if not group_id: + return + + def get_constraints(data): + maintenance_config = self._get_constraints(vnf_dict) + data['max_impacted_members'] = maintenance_config.get( + 'max_impacted_members', 1) + data['recovery_time'] = maintenance_config.get('recovery_time', 60) + + params, data = {}, {} + params['url'] = '/instance_group/%s' % group_id + if action == 'update': + data['group_id'] = group_id + data['project_id'] = vnf_dict['tenant_id'] + data['group_name'] = 'tacker_nonha_app_group_%s' % vnf_dict['id'] + data['anti_affinity_group'] = False + data['max_instances_per_host'] = 0 + data['resource_mitigation'] = True + get_constraints(data) + params.update({'data': data}) + return [params] + + @log.log + def _create_project_instance_list(self, context, vnf_dict, action): + group_id = vnf_dict.get('attributes', {}).get('maintenance_group', '') + if not group_id: + return + + params_list = [] + url = '/instance' + instances = self._get_instances(context, vnf_dict) + _instances = self._instances.get(vnf_dict['id'], {}) + if _instances: + if action == 'update': + instances = [v for v in instances if v not in _instances] + del self._instances[vnf_dict['id']] + else: + instances = [v for v in _instances if v not in instances] + if len(instances) != len(_instances): + del self._instances[vnf_dict['id']] + + if action == 'update': + maintenance_configs = self._get_constraints(vnf_dict) + for instance in instances: + params, data = {}, {} + params['url'] = '%s/%s' % (url, instance['id']) + data['project_id'] = instance['project_id'] + data['instance_id'] = instance['id'] + data['instance_name'] = instance['name'] + data['migration_type'] = maintenance_configs.get( + 'migration_type', 'MIGRATE') + data['resource_mitigation'] = maintenance_configs.get( + 'mitigation_type', True) + data['max_interruption_time'] = maintenance_configs.get( + 'max_interruption_time', + cfg.CONF.fenix.max_interruption_time) + data['lead_time'] = maintenance_configs.get( + 'lead_time', cfg.CONF.fenix.lead_time) + data['group_id'] = group_id + params.update({'data': data}) + params_list.append(params) + elif action == 'delete': + for instance in instances: + params = {} + params['url'] = '%s/%s' % (url, instance['id']) + params_list.append(params) + return params_list + + @log.log + def _get_instances(self, context, vnf_dict): + vim_res = self.vim_client.get_vim(context, vnf_dict['vim_id']) + action = '_get_instances_with_%s' % vim_res['vim_type'] + if hasattr(self, action): + return getattr(self, action)(context, vnf_dict) + return {} + + @log.log + def _get_instances_with_openstack(self, context, vnf_dict): + def get_attrs_with_link(links): + attrs = {} + for link in links: + href, rel = link['href'], link['rel'] + if rel == 'self': + words = href.split('/') + attrs['project_id'] = words[5] + attrs['stack_name'] = words[7] + break + return attrs + + instances = [] + client = self._get_openstack_clients(context, vnf_dict) + resources = client.heat.resources.list(vnf_dict['instance_id'], + nested_depth=2) + for resource in resources: + if resource.resource_type == 'OS::Nova::Server' and \ + resource.resource_status != 'DELETE_IN_PROGRESS': + instance = { + 'id': resource.physical_resource_id, + 'name': resource.resource_name + } + instance.update(get_attrs_with_link(resource.links)) + instances.append(instance) + return instances + + @log.log + def _get_scaling_policies(self, plugin, context, vnf_dict): + vnf_id = vnf_dict['id'] + policies = [] + if 'scaling_group_names' in vnf_dict['attributes']: + policies = plugin.get_vnf_policies( + context, vnf_id, filters={'type': constants.POLICY_SCALING}) + return policies + + @log.log + def _get_constraints(self, vnf, key=None, default=None): + config = vnf.get('attributes', {}).get('config', '{}') + maintenance_config = yaml.safe_load(config).get('maintenance', {}) + if key: + return maintenance_config.get(key, default) + return maintenance_config + + @log.log + def _get_openstack_clients(self, context, vnf_dict): + vim_res = self.vim_client.get_vim(context, vnf_dict['vim_id']) + region_name = vnf_dict.setdefault('placement_attr', {}).get( + 'region_name', None) + client = clients.OpenstackClients(auth_attr=vim_res['vim_auth'], + region_name=region_name) + return client diff --git a/tacker/tests/etc/samples/sample-tosca-vnfd-maintenance.yaml b/tacker/tests/etc/samples/sample-tosca-vnfd-maintenance.yaml new file mode 100644 index 000000000..e27a6dbc5 --- /dev/null +++ b/tacker/tests/etc/samples/sample-tosca-vnfd-maintenance.yaml @@ -0,0 +1,51 @@ +tosca_definitions_version: tosca_simple_profile_for_nfv_1_0_0 + +description: Maintenance VNF with Fenix + +metadata: + template_name: tosca-vnfd-maintenance + +topology_template: + node_templates: + VDU1: + capabilities: + nfv_compute: + properties: + disk_size: 15 GB + mem_size: 2048 MB + num_cpus: 2 + properties: + availability_zone: nova + image: cirros-0.4.0-x86_64-disk + maintenance: true + mgmt_driver: noop + type: tosca.nodes.nfv.VDU.Tacker + + CP11: + properties: + anti_spoofing_protection: false + management: true + order: 0 + requirements: + - virtualLink: + node: VL1 + - virtualBinding: + node: VDU1 + type: tosca.nodes.nfv.CP.Tacker + + VL1: + properties: + network_name: net_mgmt + vendor: Tacker + type: tosca.nodes.nfv.VL + policies: + - SP1: + properties: + cooldown: 120 + default_instances: 3 + increment: 1 + max_instances: 3 + min_instances: 1 + targets: + - VDU1 + type: tosca.policies.tacker.Scaling diff --git a/tacker/tests/functional/base.py b/tacker/tests/functional/base.py index 07db2c8c2..e53d05bc8 100644 --- a/tacker/tests/functional/base.py +++ b/tacker/tests/functional/base.py @@ -191,6 +191,13 @@ class BaseTackerTest(base.BaseTestCase): auth_ses = session.Session(auth=auth, verify=verify) return glance_client.Client(session=auth_ses) + @classmethod + def aodh_http_client(cls): + auth_session = cls.get_auth_session() + return SessionClient(session=auth_session, + service_type='alarming', + region_name='RegionOne') + def get_vdu_resource(self, stack_id, res_name): return self.h_client.resources.get(stack_id, res_name) diff --git a/tacker/tests/functional/vnfm/test_tosca_vnf_maintenance.py b/tacker/tests/functional/vnfm/test_tosca_vnf_maintenance.py new file mode 100644 index 000000000..7a961bff7 --- /dev/null +++ b/tacker/tests/functional/vnfm/test_tosca_vnf_maintenance.py @@ -0,0 +1,194 @@ +# Copyright 2020 Distributed Cloud and Network (DCN) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from datetime import datetime +import time +import yaml + +from oslo_serialization import jsonutils +from oslo_utils import uuidutils + +from tacker.plugins.common import constants as evt_constants +from tacker.tests import constants +from tacker.tests.functional import base +from tacker.tests.utils import read_file + + +class VnfTestMaintenanceMonitor(base.BaseTackerTest): + + def _test_vnf_tosca_maintenance(self, vnfd_file, vnf_name): + input_yaml = read_file(vnfd_file) + tosca_dict = yaml.safe_load(input_yaml) + tosca_arg = {'vnfd': {'name': vnf_name, + 'attributes': {'vnfd': tosca_dict}}} + + # Create vnfd with tosca template + vnfd_instance = self.client.create_vnfd(body=tosca_arg) + self.assertIsNotNone(vnfd_instance) + + # Create vnf with vnfd_id + vnfd_id = vnfd_instance['vnfd']['id'] + vnf_arg = {'vnf': {'vnfd_id': vnfd_id, 'name': vnf_name}} + vnf_instance = self.client.create_vnf(body=vnf_arg) + vnf_id = vnf_instance['vnf']['id'] + + self.validate_vnf_instance(vnfd_instance, vnf_instance) + + def _wait_vnf_active_and_assert_vdu_count(vdu_count, scale_type=None): + self.wait_until_vnf_active( + vnf_id, + constants.VNF_CIRROS_CREATE_TIMEOUT, + constants.ACTIVE_SLEEP_TIME) + + vnf = self.client.show_vnf(vnf_id)['vnf'] + self.assertEqual(vdu_count, len(jsonutils.loads( + vnf['mgmt_ip_address'])['VDU1'])) + + def _verify_maintenance_attributes(vnf_dict): + vnf_attrs = vnf_dict.get('attributes', {}) + maintenance_vdus = vnf_attrs.get('maintenance', '{}') + maintenance_vdus = jsonutils.loads(maintenance_vdus) + maintenance_url = vnf_attrs.get('maintenance_url', '') + words = maintenance_url.split('/') + + self.assertEqual(len(maintenance_vdus.keys()), 2) + self.assertEqual(len(words), 8) + self.assertEqual(words[5], vnf_dict['id']) + self.assertEqual(words[7], vnf_dict['tenant_id']) + + maintenance_urls = {} + for vdu, access_key in maintenance_vdus.items(): + maintenance_urls[vdu] = maintenance_url + '/' + access_key + return maintenance_urls + + def _verify_maintenance_alarm(url, project_id): + aodh_client = self.aodh_http_client() + alarm_query = { + 'and': [ + {'=': {'project_id': project_id}}, + {'=~': {'alarm_actions': url}}]} + + # Check alarm instance for MAINTENANCE_ALL + alarm_url = 'v2/query/alarms' + encoded_data = jsonutils.dumps(alarm_query) + encoded_body = jsonutils.dumps({'filter': encoded_data}) + resp, response_body = aodh_client.do_request(alarm_url, 'POST', + body=encoded_body) + self.assertEqual(len(response_body), 1) + alarm_dict = response_body[0] + self.assertEqual(url, alarm_dict.get('alarm_actions', [])[0]) + return response_body[0] + + def _verify_maintenance_actions(vnf_dict, alarm_dict): + tacker_client = self.tacker_http_client() + alarm_url = alarm_dict.get('alarm_actions', [])[0] + tacker_url = '/%s' % alarm_url[alarm_url.find('v1.0'):] + + def _request_maintenance_action(state): + alarm_body = _create_alarm_data(vnf_dict, alarm_dict, state) + resp, response_body = tacker_client.do_request( + tacker_url, 'POST', body=alarm_body) + + time.sleep(constants.SCALE_SLEEP_TIME) + target_scaled = -1 + if state == 'SCALE_IN': + target_scaled = 1 + _wait_vnf_active_and_assert_vdu_count(2, scale_type='in') + elif state == 'MAINTENANCE_COMPLETE': + target_scaled = 0 + _wait_vnf_active_and_assert_vdu_count(3, scale_type='out') + + updated_vnf = self.client.show_vnf(vnf_id)['vnf'] + scaled = updated_vnf['attributes'].get('maintenance_scaled', + '-1') + self.assertEqual(int(scaled), target_scaled) + time.sleep(constants.SCALE_WINDOW_SLEEP_TIME) + + time.sleep(constants.SCALE_WINDOW_SLEEP_TIME) + _request_maintenance_action('SCALE_IN') + _request_maintenance_action('MAINTENANCE_COMPLETE') + + self.verify_vnf_crud_events( + vnf_id, evt_constants.RES_EVT_SCALE, + evt_constants.ACTIVE, cnt=2) + self.verify_vnf_crud_events( + vnf_id, evt_constants.RES_EVT_SCALE, + evt_constants.PENDING_SCALE_OUT, cnt=1) + self.verify_vnf_crud_events( + vnf_id, evt_constants.RES_EVT_SCALE, + evt_constants.PENDING_SCALE_IN, cnt=1) + + def _create_alarm_data(vnf_dict, alarm_dict, state): + '''This function creates a raw payload of alarm to trigger Tacker directly. + + This function creates a raw payload which Fenix will put + when Fenix process maintenance procedures. Alarm_receiver and + specific steps of Fenix workflow will be tested by sending the raw + to Tacker directly. + ''' + utc_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + fake_url = 'http://localhost/' + sample_data = { + 'alarm_name': alarm_dict['name'], + 'alarm_id': alarm_dict['alarm_id'], + 'severity': 'low', + 'previous': 'alarm', + 'current': 'alarm', + 'reason': 'Alarm test for Tacker functional test', + 'reason_data': { + 'type': 'event', + 'event': { + 'message_id': uuidutils.generate_uuid(), + 'event_type': 'maintenance.scheduled', + 'generated': utc_time, + 'traits': [ + ['project_id', 1, vnf_dict['tenant_id']], + ['allowed_actions', 1, '[]'], + ['instance_ids', 1, fake_url], + ['reply_url', 1, fake_url], + ['state', 1, state], + ['session_id', 1, uuidutils.generate_uuid()], + ['actions_at', 4, utc_time], + ['reply_at', 4, utc_time], + ['metadata', 1, '{}'] + ], + 'raw': {}, + 'message_signature': uuidutils.generate_uuid() + } + } + } + return jsonutils.dumps(sample_data) + + _wait_vnf_active_and_assert_vdu_count(3) + urls = _verify_maintenance_attributes(vnf_instance['vnf']) + + maintenance_url = urls.get('ALL', '') + project_id = vnf_instance['vnf']['tenant_id'] + alarm_dict = _verify_maintenance_alarm(maintenance_url, project_id) + _verify_maintenance_actions(vnf_instance['vnf'], alarm_dict) + + try: + self.client.delete_vnf(vnf_id) + except Exception: + assert False, ( + 'Failed to delete vnf %s after the maintenance test' % vnf_id) + self.addCleanup(self.client.delete_vnfd, vnfd_id) + self.addCleanup(self.wait_until_vnf_delete, vnf_id, + constants.VNF_CIRROS_DELETE_TIMEOUT) + + def test_vnf_alarm_maintenance(self): + # instance_maintenance = self._get_instance_maintenance() + self._test_vnf_tosca_maintenance( + 'sample-tosca-vnfd-maintenance.yaml', + 'maintenance_vnf') diff --git a/tacker/tests/unit/vnfm/infra_drivers/openstack/test_vdu.py b/tacker/tests/unit/vnfm/infra_drivers/openstack/test_vdu.py index 2fa2ec9eb..b8596ffa2 100644 --- a/tacker/tests/unit/vnfm/infra_drivers/openstack/test_vdu.py +++ b/tacker/tests/unit/vnfm/infra_drivers/openstack/test_vdu.py @@ -107,6 +107,7 @@ class TestVDU(base.TestCase): cause=["Unable to reach while monitoring resource: 'VDU1'"]) self.heal_request_data_obj = heal_vnf_request.HealVnfRequest( cause='VNF monitoring fails.', + stack_id=vnf_dict['instance_id'], additional_params=[self.additional_paramas_obj]) self.heal_vdu = vdu.Vdu(self.context, vnf_dict, self.heal_request_data_obj) diff --git a/tacker/tests/unit/vnfm/test_k8s_plugin.py b/tacker/tests/unit/vnfm/test_k8s_plugin.py index 03a17f2ad..55cc765ee 100644 --- a/tacker/tests/unit/vnfm/test_k8s_plugin.py +++ b/tacker/tests/unit/vnfm/test_k8s_plugin.py @@ -32,6 +32,10 @@ class FakeCVNFMonitor(mock.Mock): pass +class FakePlugin(mock.Mock): + pass + + class FakeK8SVimClient(mock.Mock): pass @@ -44,6 +48,8 @@ class TestCVNFMPlugin(db_base.SqlTestCase): self._mock_vim_client() self._stub_get_vim() self._mock_vnf_monitor() + self._mock_vnf_maintenance_monitor() + self._mock_vnf_maintenance_plugin() self._insert_dummy_vim() self.vnfm_plugin = plugin.VNFMPlugin() mock.patch('tacker.db.common_services.common_services_db_plugin.' @@ -108,6 +114,22 @@ class TestCVNFMPlugin(db_base.SqlTestCase): self._mock( 'tacker.vnfm.monitor.VNFMonitor', fake_vnf_monitor) + def _mock_vnf_maintenance_monitor(self): + self._vnf_maintenance_mon = mock.Mock(wraps=FakeCVNFMonitor()) + fake_vnf_maintenance_monitor = mock.Mock() + fake_vnf_maintenance_monitor.return_value = self._vnf_maintenance_mon + self._mock( + 'tacker.vnfm.monitor.VNFMaintenanceAlarmMonitor', + fake_vnf_maintenance_monitor) + + def _mock_vnf_maintenance_plugin(self): + self._vnf_maintenance_plugin = mock.Mock(wraps=FakePlugin()) + fake_vnf_maintenance_plugin = mock.Mock() + fake_vnf_maintenance_plugin.return_value = self._vnf_maintenance_plugin + self._mock( + 'tacker.plugins.fenix.FenixPlugin', + fake_vnf_maintenance_plugin) + def _insert_dummy_vnf_template(self): session = self.context.session vnf_template = vnfm_db.VNFD( diff --git a/tacker/tests/unit/vnfm/test_monitor.py b/tacker/tests/unit/vnfm/test_monitor.py index 712c5d5b9..c555390e8 100644 --- a/tacker/tests/unit/vnfm/test_monitor.py +++ b/tacker/tests/unit/vnfm/test_monitor.py @@ -255,3 +255,41 @@ class TestVNFReservationAlarmMonitor(testtools.TestCase): response = test_vnf_reservation_monitor.update_vnf_with_reservation( self.plugin, self.context, vnf, policy_dict) self.assertEqual(len(response.keys()), 3) + + +class TestVNFMaintenanceAlarmMonitor(testtools.TestCase): + + def setup(self): + super(TestVNFMaintenanceAlarmMonitor, self).setUp() + + def test_process_alarm_for_vnf(self): + vnf = {'id': MOCK_VNF_ID} + trigger = {'params': {'data': { + 'alarm_id': MOCK_VNF_ID, 'current': 'alarm'}}} + test_vnf_maintenance_monitor = monitor.VNFMaintenanceAlarmMonitor() + response = test_vnf_maintenance_monitor.process_alarm_for_vnf( + vnf, trigger) + self.assertEqual(response, True) + + @mock.patch('tacker.db.common_services.common_services_db_plugin.' + 'CommonServicesPluginDb.create_event') + def test_update_vnf_with_alarm(self, mock_db_service): + mock_db_service.return_value = { + 'event_type': 'MONITOR', + 'resource_id': '9770fa22-747d-426e-9819-057a95cb778c', + 'timestamp': '2018-10-30 06:01:45.628162', + 'event_details': {'Alarm URL set successfully': { + 'start_actions': 'alarm'}}, + 'resource_state': 'CREATE', + 'id': '4583', + 'resource_type': 'vnf'} + vnf = { + 'id': MOCK_VNF_ID, + 'tenant_id': 'ad7ebc56538745a08ef7c5e97f8bd437', + 'status': 'insufficient_data'} + vdu_names = ['VDU1'] + test_vnf_maintenance_monitor = monitor.VNFMaintenanceAlarmMonitor() + response = test_vnf_maintenance_monitor.update_vnf_with_maintenance( + vnf, vdu_names) + result_keys = len(response) + len(response.get('vdus', {})) + self.assertEqual(result_keys, 4) diff --git a/tacker/tests/unit/vnfm/test_plugin.py b/tacker/tests/unit/vnfm/test_plugin.py index f954668c7..5678f08f1 100644 --- a/tacker/tests/unit/vnfm/test_plugin.py +++ b/tacker/tests/unit/vnfm/test_plugin.py @@ -49,7 +49,12 @@ class FakeDriverManager(mock.Mock): class FakeVNFMonitor(mock.Mock): - pass + def update_vnf_with_maintenance(self, vnf_dict, maintenance_vdus): + url = 'http://local:9890/v1.0/vnfs/%s/maintenance/%s' % ( + vnf_dict['id'], vnf_dict['tenant_id']) + return {'url': url, + 'vdus': {'ALL': 'ad7ebc56', + 'VDU1': '538745a0'}} class FakeGreenPool(mock.Mock): @@ -60,6 +65,10 @@ class FakeVimClient(mock.Mock): pass +class FakePlugin(mock.Mock): + pass + + class FakeException(Exception): pass @@ -143,6 +152,8 @@ class TestVNFMPlugin(db_base.SqlTestCase): self._mock_vnf_monitor() self._mock_vnf_alarm_monitor() self._mock_vnf_reservation_monitor() + self._mock_vnf_maintenance_monitor() + self._mock_vnf_maintenance_plugin() self._insert_dummy_vim() self.vnfm_plugin = plugin.VNFMPlugin() mock.patch('tacker.db.common_services.common_services_db_plugin.' @@ -219,6 +230,22 @@ class TestVNFMPlugin(db_base.SqlTestCase): 'tacker.vnfm.monitor.VNFReservationAlarmMonitor', fake_vnf_reservation_monitor) + def _mock_vnf_maintenance_monitor(self): + self._vnf_maintenance_mon = mock.Mock(wraps=FakeVNFMonitor()) + fake_vnf_maintenance_monitor = mock.Mock() + fake_vnf_maintenance_monitor.return_value = self._vnf_maintenance_mon + self._mock( + 'tacker.vnfm.monitor.VNFMaintenanceAlarmMonitor', + fake_vnf_maintenance_monitor) + + def _mock_vnf_maintenance_plugin(self): + self._vnf_maintenance_plugin = mock.Mock(wraps=FakePlugin()) + fake_vnf_maintenance_plugin = mock.Mock() + fake_vnf_maintenance_plugin.return_value = self._vnf_maintenance_plugin + self._mock( + 'tacker.plugins.fenix.FenixPlugin', + fake_vnf_maintenance_plugin) + def _insert_dummy_vnf_template(self): session = self.context.session vnf_template = vnfm_db.VNFD( @@ -1108,6 +1135,7 @@ class TestVNFMPlugin(db_base.SqlTestCase): parameter='VDU1', cause=["Unable to reach while monitoring resource: 'VDU1'"]) heal_request_data_obj = heal_vnf_request.HealVnfRequest( + stack_id=dummy_device_obj['instance_id'], cause='VNF monitoring fails.', additional_params=[additional_params_obj]) result = self.vnfm_plugin.heal_vnf(self.context, diff --git a/tacker/tosca/lib/tacker_nfv_defs.yaml b/tacker/tosca/lib/tacker_nfv_defs.yaml index e47730fe9..051f77117 100644 --- a/tacker/tosca/lib/tacker_nfv_defs.yaml +++ b/tacker/tosca/lib/tacker_nfv_defs.yaml @@ -278,6 +278,10 @@ node_types: type: tosca.datatypes.tacker.VduReservationMetadata required: false + maintenance: + type: boolean + required: false + tosca.nodes.nfv.CP.Tacker: derived_from: tosca.nodes.nfv.CP properties: diff --git a/tacker/tosca/utils.py b/tacker/tosca/utils.py index 981e4cd59..c3f222591 100644 --- a/tacker/tosca/utils.py +++ b/tacker/tosca/utils.py @@ -19,6 +19,7 @@ import yaml from collections import OrderedDict from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_utils import uuidutils from tacker._i18n import _ @@ -93,11 +94,13 @@ deletenodes = (MONITORING, FAILURE, PLACEMENT) HEAT_RESOURCE_MAP = { "flavor": "OS::Nova::Flavor", - "image": "OS::Glance::WebImage" + "image": "OS::Glance::WebImage", + "maintenance": "OS::Aodh::EventAlarm" } SCALE_GROUP_RESOURCE = "OS::Heat::AutoScalingGroup" SCALE_POLICY_RESOURCE = "OS::Heat::ScalingPolicy" +PLACEMENT_POLICY_RESOURCE = "OS::Nova::ServerGroup" @log.log @@ -258,6 +261,12 @@ def pre_process_alarm_resources(vnf, template, vdu_metadata, unique_id=None): 'before_end_actions': { 'event_type': 'lease.event.before_end_lease'}, 'end_actions': {'event_type': 'lease.event.end_lease'}} + maintenance_actions = _process_alarm_actions_for_maintenance(vnf) + if maintenance_actions: + alarm_actions.update(maintenance_actions) + alarm_resources['event_types'] = {} + alarm_resources['event_types'].update({ + 'ALL_maintenance': {'event_type': 'maintenance.scheduled'}}) alarm_resources['query_metadata'] = query_metadata alarm_resources['alarm_actions'] = alarm_actions return alarm_resources @@ -337,6 +346,22 @@ def _process_alarm_actions_for_reservation(vnf, policy): return alarm_actions +def _process_alarm_actions_for_maintenance(vnf): + # process alarm url here + alarm_actions = dict() + maintenance_props = vnf['attributes'].get('maintenance', '{}') + maintenance_props = jsonutils.loads(maintenance_props) + maintenance_url = vnf['attributes'].get('maintenance_url', '') + for vdu, access_key in maintenance_props.items(): + action = '%s_maintenance' % vdu + alarm_url = '%s/%s' % (maintenance_url.rstrip('/'), access_key) + if alarm_url: + LOG.debug('Alarm url in heat %s', alarm_url) + alarm_actions[action] = dict() + alarm_actions[action]['alarm_actions'] = [alarm_url] + return alarm_actions + + def get_volumes(template): volume_dict = dict() node_tpl = template['topology_template']['node_templates'] @@ -423,6 +448,8 @@ def add_resources_tpl(heat_dict, hot_res_tpl): "properties": {} } + if res == "maintenance": + continue for prop, val in (vdu_dict).items(): # change from 'get_input' to 'get_param' to meet HOT template if isinstance(val, dict): @@ -517,6 +544,7 @@ def post_process_heat_template(heat_tpl, mgmt_ports, metadata, if heat_dict['resources'].get(vdu_name): heat_dict['resources'][vdu_name]['properties']['metadata'] =\ metadata_dict + add_resources_tpl(heat_dict, res_tpl) query_metadata = alarm_resources.get('query_metadata') alarm_actions = alarm_resources.get('alarm_actions') @@ -532,16 +560,14 @@ def post_process_heat_template(heat_tpl, mgmt_ports, metadata, if alarm_actions: for trigger_name, alarm_actions_dict in alarm_actions.items(): if heat_dict['resources'].get(trigger_name): - heat_dict['resources'][trigger_name]['properties']. \ - update(alarm_actions_dict) - + heat_dict['resources'][trigger_name]['properties'].update( + alarm_actions_dict) if event_types: for trigger_name, event_type in event_types.items(): if heat_dict['resources'].get(trigger_name): heat_dict['resources'][trigger_name]['properties'].update( event_type) - add_resources_tpl(heat_dict, res_tpl) for res in heat_dict["resources"].values(): if not res['type'] == HEAT_SOFTWARE_CONFIG: continue @@ -1038,6 +1064,15 @@ def findvdus(template): return vdus +def find_maintenance_vdus(template): + maintenance_vdu_names = list() + vdus = findvdus(template) + for nt in vdus: + if nt.get_properties().get('maintenance'): + maintenance_vdu_names.append(nt.name) + return maintenance_vdu_names + + def get_flavor_dict(template, flavor_extra_input=None): flavor_dict = {} vdus = findvdus(template) @@ -1152,6 +1187,27 @@ def get_resources_dict(template, flavor_extra_input=None): return res_dict +def add_maintenance_resources(template, res_tpl): + res_dict = {} + maintenance_vdus = find_maintenance_vdus(template) + maintenance_vdus.append('ALL') + if maintenance_vdus: + for vdu_name in maintenance_vdus: + res_dict[vdu_name] = {} + res_tpl['maintenance'] = res_dict + + +@log.log +def get_policy_dict(template, policy_type): + policy_dict = dict() + for policy in template.policies: + if (policy.type_definition.is_derived_from(policy_type)): + policy_attrs = dict() + policy_attrs['targets'] = policy.targets + policy_dict[policy.name] = policy_attrs + return policy_dict + + @log.log def get_scaling_policy(template): scaling_policy_names = list() diff --git a/tacker/vnfm/infra_drivers/openstack/openstack.py b/tacker/vnfm/infra_drivers/openstack/openstack.py index a632ed81f..0d4c0a0e9 100644 --- a/tacker/vnfm/infra_drivers/openstack/openstack.py +++ b/tacker/vnfm/infra_drivers/openstack/openstack.py @@ -427,12 +427,24 @@ class OpenStack(abstract_driver.VnfAbstractDriver, @log.log def heal_wait(self, plugin, context, vnf_dict, auth_attr, region_name=None): - stack = self._wait_until_stack_ready(vnf_dict['instance_id'], + region_name = vnf_dict.get('placement_attr', {}).get( + 'region_name', None) + heatclient = hc.HeatClient(auth_attr, region_name) + stack_id = vnf_dict.get('heal_stack_id', vnf_dict['instance_id']) + + stack = self._wait_until_stack_ready(stack_id, auth_attr, infra_cnst.STACK_UPDATE_IN_PROGRESS, infra_cnst.STACK_UPDATE_COMPLETE, vnfm.VNFHealWaitFailed, region_name=region_name) - - mgmt_ips = self._find_mgmt_ips(stack.outputs) + # scaling enabled + if vnf_dict['attributes'].get('scaling_group_names'): + group_names = jsonutils.loads( + vnf_dict['attributes'].get('scaling_group_names')).values() + mgmt_ips = self._find_mgmt_ips_from_groups(heatclient, + vnf_dict['instance_id'], + group_names) + else: + mgmt_ips = self._find_mgmt_ips(stack.outputs) if mgmt_ips: vnf_dict['mgmt_ip_address'] = jsonutils.dump_as_bytes(mgmt_ips) @@ -462,10 +474,13 @@ class OpenStack(abstract_driver.VnfAbstractDriver, return mgmt_ips mgmt_ips = {} + ignore_status = ['DELETE_COMPLETE', 'DELETE_IN_PROGRESS'] for group_name in group_names: # Get scale group grp = heat_client.resource_get(instance_id, group_name) for rsc in heat_client.resource_get_list(grp.physical_resource_id): + if rsc.resource_status in ignore_status: + continue # Get list of resources in scale group scale_rsc = heat_client.resource_get(grp.physical_resource_id, rsc.resource_name) diff --git a/tacker/vnfm/infra_drivers/openstack/translate_template.py b/tacker/vnfm/infra_drivers/openstack/translate_template.py index d4754ae82..a1ed66d0c 100644 --- a/tacker/vnfm/infra_drivers/openstack/translate_template.py +++ b/tacker/vnfm/infra_drivers/openstack/translate_template.py @@ -338,6 +338,10 @@ class TOSCAToHOT(object): heat_template_yaml, scaling_policy_names) self.vnf['attributes']['scaling_group_names'] =\ jsonutils.dump_as_bytes(scaling_group_dict) + + if self.vnf['attributes'].get('maintenance', None): + toscautils.add_maintenance_resources(tosca, res_tpl) + heat_template_yaml = toscautils.post_process_heat_template( heat_template_yaml, mgmt_ports, metadata, alarm_resources, res_tpl, block_storage_details, self.unsupported_props, diff --git a/tacker/vnfm/infra_drivers/openstack/vdu.py b/tacker/vnfm/infra_drivers/openstack/vdu.py index e0b52d125..e5d66cf4e 100644 --- a/tacker/vnfm/infra_drivers/openstack/vdu.py +++ b/tacker/vnfm/infra_drivers/openstack/vdu.py @@ -33,6 +33,7 @@ class Vdu(object): self.context = context self.vnf_dict = vnf_dict self.heal_request_data_obj = heal_request_data_obj + self.stack_id = self.heal_request_data_obj.stack_id vim_id = self.vnf_dict['vim_id'] vim_res = vim_client.VimClient().get_vim(context, vim_id) placement_attr = vnf_dict.get('placement_attr', {}) @@ -53,15 +54,15 @@ class Vdu(object): additional_params = self.heal_request_data_obj.additional_params for additional_param in additional_params: resource_name = additional_param.parameter - res_status = self._get_resource_status( - self.vnf_dict['instance_id'], resource_name) + res_status = self._get_resource_status(self.stack_id, + resource_name) if res_status != 'CHECK_FAILED': self.heat_client.resource_mark_unhealthy( - stack_id=self.vnf_dict['instance_id'], + stack_id=self.stack_id, resource_name=resource_name, mark_unhealthy=True, resource_status_reason=additional_param.cause) LOG.debug("Heat stack '%s' resource '%s' marked as " - "unhealthy", self.vnf_dict['instance_id'], + "unhealthy", self.stack_id, resource_name) evt_details = (("HealVnfRequest invoked to mark resource " "'%s' to unhealthy.") % resource_name) @@ -70,7 +71,7 @@ class Vdu(object): evt_details) else: LOG.debug("Heat stack '%s' resource '%s' already mark " - "unhealthy.", self.vnf_dict['instance_id'], + "unhealthy.", self.stack_id, resource_name) def heal_vdu(self): @@ -81,11 +82,11 @@ class Vdu(object): # Mark all the resources as unhealthy self._resource_mark_unhealthy() - self.heat_client.update(stack_id=self.vnf_dict['instance_id'], + self.heat_client.update(stack_id=self.stack_id, existing=True) LOG.debug("Heat stack '%s' update initiated to revive " - "unhealthy resources.", self.vnf_dict['instance_id']) + "unhealthy resources.", self.stack_id) evt_details = (("HealVnfRequest invoked to update the stack " - "'%s'") % self.vnf_dict['instance_id']) + "'%s'") % self.stack_id) vnfm_utils.log_events(self.context, self.vnf_dict, constants.RES_EVT_HEAL, evt_details) diff --git a/tacker/vnfm/monitor.py b/tacker/vnfm/monitor.py index 47e17547a..8defa8bde 100644 --- a/tacker/vnfm/monitor.py +++ b/tacker/vnfm/monitor.py @@ -17,6 +17,8 @@ import ast import copy import inspect +import random +import string import threading import time @@ -427,3 +429,40 @@ class VNFReservationAlarmMonitor(VNFAlarmMonitor): alarm_dict['status'] = params['data'].get('current') driver = 'ceilometer' return self.process_alarm(driver, vnf, alarm_dict) + + +class VNFMaintenanceAlarmMonitor(VNFAlarmMonitor): + """VNF Maintenance Alarm monitor""" + + def update_vnf_with_maintenance(self, vnf, vdu_names): + maintenance = dict() + vdus = dict() + params = dict() + params['vnf_id'] = vnf['id'] + params['mon_policy_name'] = 'maintenance' + params['mon_policy_action'] = vnf['tenant_id'] + driver = 'ceilometer' + + url = self.call_alarm_url(driver, vnf, params) + maintenance['url'] = url[:url.rindex('/')] + vdu_names.append('ALL') + for vdu in vdu_names: + access_key = ''.join( + random.SystemRandom().choice( + string.ascii_lowercase + string.digits) + for _ in range(8)) + vdus[vdu] = access_key + maintenance.update({'vdus': vdus}) + details = "Alarm URL set successfully: %s" % maintenance['url'] + vnfm_utils.log_events(t_context.get_admin_context(), vnf, + constants.RES_EVT_MONITOR, details) + return maintenance + + def process_alarm_for_vnf(self, vnf, trigger): + """call in plugin""" + params = trigger['params'] + alarm_dict = dict() + alarm_dict['alarm_id'] = params['data'].get('alarm_id') + alarm_dict['status'] = params['data'].get('current') + driver = 'ceilometer' + return self.process_alarm(driver, vnf, alarm_dict) diff --git a/tacker/vnfm/plugin.py b/tacker/vnfm/plugin.py index 22a0f7ecc..0f461faf6 100644 --- a/tacker/vnfm/plugin.py +++ b/tacker/vnfm/plugin.py @@ -21,6 +21,7 @@ import yaml import eventlet from oslo_config import cfg from oslo_log import log as logging +from oslo_serialization import jsonutils from oslo_utils import excutils from oslo_utils import uuidutils from toscaparser.tosca_template import ToscaTemplate @@ -34,6 +35,7 @@ from tacker import context as t_context from tacker.db.vnfm import vnfm_db from tacker.extensions import vnfm from tacker.plugins.common import constants +from tacker.plugins import fenix from tacker.tosca import utils as toscautils from tacker.vnfm.mgmt_drivers import constants as mgmt_constants from tacker.vnfm import monitor @@ -147,7 +149,9 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): self._vnf_monitor = monitor.VNFMonitor(self.boot_wait) self._vnf_alarm_monitor = monitor.VNFAlarmMonitor() self._vnf_reservation_monitor = monitor.VNFReservationAlarmMonitor() + self._vnf_maintenance_monitor = monitor.VNFMaintenanceAlarmMonitor() self._vnf_app_monitor = monitor.VNFAppMonitor() + self._vnf_maintenance_plugin = fenix.FenixPlugin() self._init_monitoring() def _init_monitoring(self): @@ -258,6 +262,13 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): vnfd_dict = yaml.safe_load(vnfd_yaml) if not (vnfd_dict and vnfd_dict.get('tosca_definitions_version')): return + try: + toscautils.updateimports(vnfd_dict) + tosca_vnfd = ToscaTemplate(a_file=False, + yaml_dict_tpl=vnfd_dict) + except Exception as e: + LOG.exception("tosca-parser error: %s", str(e)) + raise vnfm.ToscaParserFailed(error_msg_details=str(e)) polices = vnfd_dict['topology_template'].get('policies', []) for policy_dict in polices: name, policy = list(policy_dict.items())[0] @@ -273,6 +284,13 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): self, context, vnf_dict, policy) vnf_dict['attributes']['reservation_policy'] = vnf_dict['id'] vnf_dict['attributes'].update(alarm_url) + maintenance_vdus = toscautils.find_maintenance_vdus(tosca_vnfd) + maintenance = \ + self._vnf_maintenance_monitor.update_vnf_with_maintenance( + vnf_dict, maintenance_vdus) + vnf_dict['attributes'].update({ + 'maintenance': jsonutils.dumps(maintenance['vdus'])}) + vnf_dict['attributes']['maintenance_url'] = maintenance['url'] def add_vnf_to_appmonitor(self, context, vnf_dict): appmonitor = self._vnf_app_monitor.create_app_dict(context, vnf_dict) @@ -281,6 +299,8 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): def config_vnf(self, context, vnf_dict): config = vnf_dict['attributes'].get('config') if not config: + self._vnf_maintenance_plugin.create_vnf_constraints(self, context, + vnf_dict) return if isinstance(config, str): # TODO(dkushwaha) remove this load once db supports storing @@ -367,6 +387,7 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): if driver_name == 'openstack': self.mgmt_create_pre(context, vnf_dict) self.add_alarm_url_to_vnf(context, vnf_dict) + vnf_dict['attributes']['maintenance_group'] = uuidutils.generate_uuid() try: instance_id = self._vnf_manager.invoke( @@ -431,9 +452,9 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): if 'app_monitoring_policy' in vnf_dict['attributes']: self.add_vnf_to_appmonitor(context, vnf_dict) - if vnf_dict['status'] is not constants.ERROR: self.add_vnf_to_monitor(context, vnf_dict) + self.config_vnf(context, vnf_dict) self.spawn_n(create_vnf_wait) return vnf_dict @@ -466,15 +487,18 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): with excutils.save_and_reraise_exception(): new_status = constants.ERROR self._vnf_monitor.delete_hosting_vnf(vnf_dict['id']) + self._vnf_maintenance_plugin.post(context, vnf_dict) self.set_vnf_error_status_reason(context, vnf_dict['id'], six.text_type(e)) except exceptions.MgmtDriverException as e: LOG.error('VNF configuration failed') new_status = constants.ERROR self._vnf_monitor.delete_hosting_vnf(vnf_dict['id']) + self._vnf_maintenance_plugin.post(context, vnf_dict) self.set_vnf_error_status_reason(context, vnf_dict['id'], six.text_type(e)) + del vnf_dict['heal_stack_id'] vnf_dict['status'] = new_status self.mgmt_update_post(context, vnf_dict) @@ -482,6 +506,9 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): evt_details = ("Ends the heal vnf request for VNF '%s'" % vnf_dict['id']) self._vnf_monitor.update_hosting_vnf(vnf_dict, evt_details) + self._vnf_maintenance_plugin.update_vnf_instances(self, context, + vnf_dict) + self._vnf_maintenance_plugin.post(context, vnf_dict) # _update_vnf_post() method updates vnf_status and mgmt_ip_address self._update_vnf_post(context, vnf_dict['id'], new_status, vnf_dict, @@ -521,6 +548,8 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): self._update_vnf_post(context, vnf_dict['id'], new_status, vnf_dict, constants.PENDING_UPDATE, constants.RES_EVT_UPDATE) + self._vnf_maintenance_plugin.create_vnf_constraints(self, context, + vnf_dict) def update_vnf(self, context, vnf_id, vnf): vnf_attributes = vnf['vnf']['attributes'] @@ -591,6 +620,9 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): self._vnf_monitor.update_hosting_vnf(vnf_dict, evt_details) try: + vnf_dict['heal_stack_id'] = heal_request_data_obj.stack_id + self._vnf_maintenance_plugin.project_instance_pre(context, + vnf_dict) self.mgmt_update_pre(context, vnf_dict) self._vnf_manager.invoke( driver_name, 'heal_vdu', plugin=self, @@ -604,6 +636,7 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): vnf_dict['id'], six.text_type(e)) self.mgmt_update_post(context, vnf_dict) + self._vnf_maintenance_plugin.post(context, vnf_dict) self._update_vnf_post(context, vnf_id, constants.ERROR, vnf_dict, constants.PENDING_HEAL, @@ -637,6 +670,8 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): self.set_vnf_error_status_reason(context, vnf_dict['id'], vnf_dict['error_reason']) + self._vnf_maintenance_plugin.delete_vnf_constraints(self, context, + vnf_dict) self.mgmt_delete_post(context, vnf_dict) self._delete_vnf_post(context, vnf_dict, e) @@ -654,6 +689,8 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): mgmt_constants.KEY_KWARGS: {'vnf': vnf_dict}, } try: + self._vnf_maintenance_plugin.project_instance_pre(context, + vnf_dict) self.mgmt_delete_pre(context, vnf_dict) self.mgmt_call(context, vnf_dict, kwargs) if instance_id: @@ -737,6 +774,8 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): LOG.debug("Policy %(policy)s vnf is at %(status)s", {'policy': policy['name'], 'status': status}) + self._vnf_maintenance_plugin.project_instance_pre(context, + result) return result # post @@ -750,6 +789,10 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): LOG.debug("Policy %(policy)s vnf is at %(status)s", {'policy': policy['name'], 'status': new_status}) + action = 'delete' if policy['action'] == 'in' else 'update' + self._vnf_maintenance_plugin.update_vnf_instances(self, context, + result, + action=action) return result # action @@ -1040,3 +1083,20 @@ class VNFMPlugin(vnfm_db.VNFMPluginDb, VNFMMgmtMixin): else: raise vnfm.VNFInactive(vnf_id=vnf_id, message=_(' Cannot fetch details')) + + def create_vnf_maintenance(self, context, vnf_id, maintenance): + _maintenance = self._vnf_maintenance_plugin.validate_maintenance( + maintenance.copy()) + vnf = self.get_vnf(context, vnf_id) + _maintenance['vnf'] = vnf + self._vnf_maintenance_plugin.handle_maintenance( + self, context, _maintenance) + policy_action = _maintenance.get('policy_action', '') + if policy_action: + self._vnf_action.invoke( + policy_action['action'], 'execute_action', plugin=self, + context=context, vnf_dict=vnf, args=policy_action['args']) + else: + self._vnf_maintenance_plugin.request(self, context, vnf, + _maintenance) + return maintenance['maintenance'] diff --git a/tacker/vnfm/policy_actions/vdu_autoheal/vdu_autoheal.py b/tacker/vnfm/policy_actions/vdu_autoheal/vdu_autoheal.py index 37308f4c6..d53733e2a 100644 --- a/tacker/vnfm/policy_actions/vdu_autoheal/vdu_autoheal.py +++ b/tacker/vnfm/policy_actions/vdu_autoheal/vdu_autoheal.py @@ -34,6 +34,9 @@ class VNFActionVduAutoheal(abstract_action.AbstractPolicyAction): def execute_action(self, plugin, context, vnf_dict, args): vdu_name = args.get('vdu_name') + stack_id = args.get('stack_id', vnf_dict['instance_id']) + heat_tpl = args.get('heat_tpl', 'heat_template') + cause = args.get('cause', []) if vdu_name is None: LOG.error("VDU resource of vnf '%s' is not present for " "autoheal." % vnf_dict['id']) @@ -46,7 +49,7 @@ class VNFActionVduAutoheal(abstract_action.AbstractPolicyAction): """ resource_list = [vdu_name] heat_template = yaml.safe_load(vnf_dict['attributes'].get( - 'heat_template')) + heat_tpl)) vdu_resources = heat_template['resources'].get(vdu_name) cp_resources = vdu_resources['properties'].get('networks') for resource in cp_resources: @@ -54,17 +57,18 @@ class VNFActionVduAutoheal(abstract_action.AbstractPolicyAction): return resource_list + if not cause or type(cause) is not list: + cause = ["Unable to reach while monitoring resource: '%s'", + "Failed to monitor VDU resource '%s'"] resource_list = _get_vdu_resources() additional_params = [] for resource in resource_list: - additional_paramas_obj = objects.HealVnfAdditionalParams( - parameter=resource, - cause=["Unable to reach while monitoring resource: '%s'" % - resource]) - additional_params.append(additional_paramas_obj) + additional_params_obj = objects.HealVnfAdditionalParams( + parameter=resource, cause=[cause[0] % resource]) + additional_params.append(additional_params_obj) heal_request_data_obj = objects.HealVnfRequest( - cause=("Failed to monitor VDU resource '%s'" % vdu_name), - additional_params=additional_params) + stack_id=stack_id, + cause=(cause[-1] % vdu_name), additional_params=additional_params) plugin.heal_vnf(context, vnf_dict['id'], heal_request_data_obj)