add NFVFaultMgmtAPI to get alarms from fm in pods

after fm containerization, nfvi would raise openstack
alarms to the fm in pods. while retrieve both platform
and openstack alarms are still required.

So here we add NFVFaultMgmtAPI to get alarms from fm
in pods and gather all alarms in vim strategy steps.

add param "fault_management_pod_disabled" with 'True' as
the default value into config file to disable nfvi get
alarms from containerized fm. it will be removed in future

Depends-On: https://review.opendev.org/658972/
Depends-On: https://review.opendev.org/662896/
Story: 2004008
Task: 30930
Change-Id: I79e0010f556f99923df25f355e554f43283d26f2
Signed-off-by: SidneyAn <ran1.an@intel.com>

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/nfvi_fault_mgmt_api.py

@@ -0,0 +1,221 @@
+#
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+from six.moves import http_client as httplib
+
+from nfv_common import debug
+from nfv_plugins.nfvi_plugins import config
+from nfv_plugins.nfvi_plugins.openstack import exceptions
+from nfv_plugins.nfvi_plugins.openstack import fm
+from nfv_plugins.nfvi_plugins.openstack.objects import OPENSTACK_SERVICE
+from nfv_plugins.nfvi_plugins.openstack import openstack
+from nfv_vim import nfvi
+
+DLOG = debug.debug_get_logger('nfv_plugins.nfvi_plugins.fault_mgmt_api')
+
+
+class NFVIFaultMgmtAPI(nfvi.api.v1.NFVIFaultMgmtAPI):
+    """
+    NFV Fault Management API Class Definition
+    """
+    _name = 'Fault-Management-API'
+    _version = '1.0.0'
+    _provider = 'StarlingX'
+    _signature = '2808f351-92bb-482c-b873-66ab232254af'
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def version(self):
+        return self._version
+
+    @property
+    def provider(self):
+        return self._provider
+
+    @property
+    def signature(self):
+        return self._signature
+
+    def __init__(self):
+        super(NFVIFaultMgmtAPI, self).__init__()
+        self._openstack_token = None
+        self._openstack_directory = None
+
+    def get_openstack_alarms(self, future, callback):
+        """
+        Get alarms
+        """
+        response = dict()
+        response['completed'] = False
+        response['reason'] = ''
+
+        try:
+            future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
+
+            if self._openstack_token is None or \
+                    self._openstack_token.is_expired():
+                future.work(openstack.get_token, self._openstack_directory)
+                future.result = (yield)
+
+                if not future.result.is_complete() or \
+                        future.result.data is None:
+                    DLOG.error("OpenStack get-token did not complete.")
+                    return
+
+                self._openstack_token = future.result.data
+
+            future.work(fm.get_alarms, self._openstack_token, OPENSTACK_SERVICE.FM)
+            future.result = (yield)
+
+            if not future.result.is_complete():
+                return
+
+            alarms = list()
+
+            for alarm_data in future.result.data['alarms']:
+                alarm = nfvi.objects.v1.Alarm(
+                    alarm_data['uuid'], alarm_data['alarm_id'],
+                    alarm_data['entity_instance_id'], alarm_data['severity'],
+                    alarm_data['reason_text'], alarm_data['timestamp'],
+                    alarm_data['mgmt_affecting'])
+                alarms.append(alarm)
+
+            response['result-data'] = alarms
+            response['completed'] = True
+
+        except exceptions.OpenStackRestAPIException as e:
+            if httplib.UNAUTHORIZED == e.http_status_code:
+                response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
+                if self._openstack_token is not None:
+                    self._openstack_token.set_expired()
+
+            else:
+                DLOG.exception("Caught exception while trying to get alarms, "
+                               "error=%s." % e)
+
+        except Exception as e:
+            DLOG.exception("Caught exception while trying to get alarms, "
+                           "error=%s." % e)
+
+        finally:
+            callback.send(response)
+            callback.close()
+
+    def get_openstack_logs(self, future, start_period, end_period, callback):
+        """
+        Get logs
+        """
+        response = dict()
+        response['completed'] = False
+        response['reason'] = ''
+
+        try:
+            future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
+
+            if self._openstack_token is None or \
+                    self._openstack_token.is_expired():
+                future.work(openstack.get_token, self._openstack_directory)
+                future.result = (yield)
+
+                if not future.result.is_complete() or \
+                        future.result.data is None:
+                    DLOG.error("OpenStack get-token did not complete.")
+                    return
+
+                self._openstack_token = future.result.data
+
+            future.work(fm.get_logs, self._openstack_token, start_period,
+                        end_period)
+            future.result = (yield)
+
+            if not future.result.is_complete():
+                return
+
+            response['completed'] = True
+
+        except exceptions.OpenStackRestAPIException as e:
+            if httplib.UNAUTHORIZED == e.http_status_code:
+                response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
+                if self._openstack_token is not None:
+                    self._openstack_token.set_expired()
+
+            else:
+                DLOG.exception("Caught exception while trying to get logs, "
+                               "error=%s." % e)
+
+        except Exception as e:
+            DLOG.exception("Caught exception while trying to get logs, "
+                           "error=%s." % e)
+
+        finally:
+            callback.send(response)
+            callback.close()
+
+    def get_openstack_alarm_history(self, future, start_period, end_period, callback):
+        """
+        Get alarm history
+        """
+        response = dict()
+        response['completed'] = False
+        response['reason'] = ''
+
+        try:
+            future.set_timeouts(config.CONF.get('nfvi-timeouts', None))
+
+            if self._openstack_token is None or \
+                    self._openstack_token.is_expired():
+                future.work(openstack.get_token, self._openstack_directory)
+                future.result = (yield)
+
+                if not future.result.is_complete() or \
+                        future.result.data is None:
+                    DLOG.error("OpenStack get-token did not complete.")
+                    return
+
+                self._openstack_token = future.result.data
+
+            future.work(fm.get_alarm_history, self._openstack_token,
+                        start_period, end_period)
+            future.result = (yield)
+
+            if not future.result.is_complete():
+                return
+
+            response['completed'] = True
+
+        except exceptions.OpenStackRestAPIException as e:
+            if httplib.UNAUTHORIZED == e.http_status_code:
+                response['error-code'] = nfvi.NFVI_ERROR_CODE.TOKEN_EXPIRED
+                if self._openstack_token is not None:
+                    self._openstack_token.set_expired()
+
+            else:
+                DLOG.exception("Caught exception while trying to get alarm "
+                               "history, error=%s." % e)
+
+        except Exception as e:
+            DLOG.exception("Caught exception while trying to get alarm "
+                           "history, error=%s." % e)
+
+        finally:
+            callback.send(response)
+            callback.close()
+
+    def initialize(self, config_file):
+        """
+        Initialize the plugin
+        """
+        config.load(config_file)
+        self._openstack_directory = openstack.get_directory(
+            config, openstack.SERVICE_CATEGORY.OPENSTACK)
+
+    def finalize(self):
+        """
+        Finalize the plugin
+        """
+        pass

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/fm.py

@@ -12,11 +12,11 @@ from nfv_plugins.nfvi_plugins.openstack.rest_api import rest_api_request
 DLOG = debug.debug_get_logger('nfv_plugins.nfvi_plugins.openstack.fm')
 
 
-def get_alarms(token):
+def get_alarms(token, fm_service=PLATFORM_SERVICE.FM):
     """
     Asks Fault Management for customer alarms
     """
-    url = token.get_service_url(PLATFORM_SERVICE.FM)
+    url = token.get_service_url(fm_service)
     if url is None:
         raise ValueError("OpenStack FM URL is invalid")
 
@@ -26,11 +26,11 @@ def get_alarms(token):
     return response
 
 
-def get_logs(token, start=None, end=None):
+def get_logs(token, start=None, end=None, fm_service=PLATFORM_SERVICE.FM):
     """
     Asks Fault Management for customer logs
     """
-    url = token.get_service_url(PLATFORM_SERVICE.FM)
+    url = token.get_service_url(fm_service)
     if url is None:
         raise ValueError("OpenStack FM URL is invalid")
 
@@ -55,11 +55,11 @@ def get_logs(token, start=None, end=None):
     return response
 
 
-def get_alarm_history(token, start=None, end=None):
+def get_alarm_history(token, start=None, end=None, fm_service=PLATFORM_SERVICE.FM):
     """
     Asks Fault Management for customer alarm history
     """
-    url = token.get_service_url(PLATFORM_SERVICE.FM)
+    url = token.get_service_url(fm_service)
     if url is None:
         raise ValueError("OpenStack FM URL is invalid")
 

nfv/nfv-plugins/nfv_plugins/nfvi_plugins/openstack/objects.py

@@ -58,6 +58,7 @@ class OpenStackServices(Constants):
     NEUTRON = Constant('neutron')
     NOVA = Constant('nova')
     HEAT = Constant('heat')
+    FM = Constant('fm')
 
 
 # OpenStack Services Constant

nfv/nfv-plugins/setup.py

@@ -34,6 +34,7 @@ setup(
             'compute_plugin = nfv_plugins.nfvi_plugins.nfvi_compute_api:NFVIComputeAPI',
             'guest_plugin = nfv_plugins.nfvi_plugins.nfvi_guest_api:NFVIGuestAPI',
             'sw_mgmt_plugin = nfv_plugins.nfvi_plugins.nfvi_sw_mgmt_api:NFVISwMgmtAPI',
+            'fault_mgmt_plugin = nfv_plugins.nfvi_plugins.nfvi_fault_mgmt_api:NFVIFaultMgmtAPI',
         ],
     }
 )

nfv/nfv-tests/nfv_scenario_tests/tests/_test_hosts.py

@@ -196,7 +196,6 @@ class TestHost(_test_base.Test):
         Fetch the customer alarms raised
         """
         self._customer_alarms = fm.get_alarms(self.platform_token).result_data
-
         return
 
     def _refresh_customer_logs(self):

nfv/nfv-vim/nfv_vim/nfvi/__init__.py

@@ -58,6 +58,11 @@ from nfv_vim.nfvi._nfvi_compute_module import nfvi_unpause_instance  # noqa: F40
 
 from nfv_vim.nfvi._nfvi_defs import NFVI_ERROR_CODE  # noqa: F401
 
+from nfv_vim.nfvi._nfvi_fault_mgmt_module import nfvi_fault_mgmt_plugin_disabled  # noqa: F401
+from nfv_vim.nfvi._nfvi_fault_mgmt_module import nfvi_get_openstack_alarm_history  # noqa: F401
+from nfv_vim.nfvi._nfvi_fault_mgmt_module import nfvi_get_openstack_alarms  # noqa: F401
+from nfv_vim.nfvi._nfvi_fault_mgmt_module import nfvi_get_openstack_logs  # noqa: F401
+
 from nfv_vim.nfvi._nfvi_guest_module import nfvi_create_guest_host_services  # noqa: F401
 from nfv_vim.nfvi._nfvi_guest_module import nfvi_delete_guest_host_services  # noqa: F401
 from nfv_vim.nfvi._nfvi_guest_module import nfvi_disable_guest_host_services  # noqa: F401

nfv/nfv-vim/nfv_vim/nfvi/_nfvi_fault_mgmt_module.py

@@ -0,0 +1,62 @@
+#
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+from nfv_common import debug
+from nfv_vim.nfvi._nfvi_fault_mgmt_plugin import NFVIFaultMgmtPlugin
+
+DLOG = debug.debug_get_logger('nfv_vim.nfvi.nfvi_fault_mgmt_module')
+
+_fault_mgmt_plugin = None
+
+
+def nfvi_fault_mgmt_plugin_disabled():
+    """
+    Get fault management plugin disabled status
+    """
+    return (_fault_mgmt_plugin is None)
+
+
+def nfvi_get_openstack_alarms(callback):
+    """
+    Get alarms
+    """
+    cmd_id = _fault_mgmt_plugin.invoke_plugin('get_openstack_alarms', callback=callback)
+    return cmd_id
+
+
+def nfvi_get_openstack_logs(start_period, end_period, callback):
+    """
+    Get logs
+    """
+    cmd_id = _fault_mgmt_plugin.invoke_plugin('get_openstack_logs', start_period,
+                                              end_period, callback=callback)
+    return cmd_id
+
+
+def nfvi_get_openstack_alarm_history(start_period, end_period, callback):
+    """
+    Get logs
+    """
+    cmd_id = _fault_mgmt_plugin.invoke_plugin('get_openstack_alarm_history', start_period,
+                                              end_period, callback=callback)
+    return cmd_id
+
+
+def nfvi_fault_mgmt_initialize(config, pool):
+    """
+    Initialize the NFVI fault_mgmt package
+    """
+    global _fault_mgmt_plugin
+
+    _fault_mgmt_plugin = NFVIFaultMgmtPlugin(config['namespace'], pool)
+    _fault_mgmt_plugin.initialize(config['config_file'])
+
+
+def nfvi_fault_mgmt_finalize():
+    """
+    Finalize the NFVI fault_mgmt package
+    """
+    if _fault_mgmt_plugin is not None:
+        _fault_mgmt_plugin.finalize()

nfv/nfv-vim/nfv_vim/nfvi/_nfvi_fault_mgmt_plugin.py

@@ -0,0 +1,29 @@
+#
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+from nfv_common import debug
+from nfv_common import tasks
+
+from nfv_vim.nfvi._nfvi_plugin import NFVIPlugin
+
+DLOG = debug.debug_get_logger('nfv_vim.nfvi.nfvi_fault_mgmt_plugin')
+
+
+class NFVIFaultMgmtPlugin(NFVIPlugin):
+    """
+    NFVI Fault Management Plugin
+    """
+    _version = '1.0.0'
+    _signature = '2808f351-92bb-482c-b873-66ab232254af'
+    _plugin_type = 'fault_mgmt_plugin'
+
+    def __init__(self, namespace, pool):
+        scheduler = tasks.TaskScheduler('fault_mgmt_plugin', pool)
+        super(NFVIFaultMgmtPlugin, self).__init__(
+            namespace,
+            NFVIFaultMgmtPlugin._version,
+            NFVIFaultMgmtPlugin._signature,
+            NFVIFaultMgmtPlugin._plugin_type,
+            scheduler)

nfv/nfv-vim/nfv_vim/nfvi/_nfvi_module.py

@@ -10,6 +10,8 @@ from nfv_vim.nfvi._nfvi_block_storage_module import nfvi_block_storage_finalize
 from nfv_vim.nfvi._nfvi_block_storage_module import nfvi_block_storage_initialize
 from nfv_vim.nfvi._nfvi_compute_module import nfvi_compute_finalize
 from nfv_vim.nfvi._nfvi_compute_module import nfvi_compute_initialize
+from nfv_vim.nfvi._nfvi_fault_mgmt_module import nfvi_fault_mgmt_finalize
+from nfv_vim.nfvi._nfvi_fault_mgmt_module import nfvi_fault_mgmt_initialize
 from nfv_vim.nfvi._nfvi_guest_module import nfvi_guest_finalize
 from nfv_vim.nfvi._nfvi_guest_module import nfvi_guest_initialize
 from nfv_vim.nfvi._nfvi_identity_module import nfvi_identity_finalize
@@ -48,6 +50,12 @@ def nfvi_initialize(config):
                                           'False') in DISABLED_LIST)
     guest_plugin_disabled = (config.get('guest_plugin_disabled',
                                         'False') in DISABLED_LIST)
+    # 'fault_management_pod_disabled' is used to disable get alarms
+    # from containerized fm and will be removed in future.
+    fault_mgmt_plugin_disabled = (config.get('fault_mgmt_plugin_disabled',
+                                        'False') in DISABLED_LIST) or \
+                                 (config.get('fault_management_pod_disabled',
+                                        'True') in DISABLED_LIST)
 
     _task_worker_pools['identity'] = \
         tasks.TaskWorkerPool('Identity', num_workers=1)
@@ -89,6 +97,11 @@ def nfvi_initialize(config):
         tasks.TaskWorkerPool('Sw-Mgmt', num_workers=1)
     nfvi_sw_mgmt_initialize(config, _task_worker_pools['sw_mgmt'])
 
+    if not fault_mgmt_plugin_disabled:
+        _task_worker_pools['fault_mgmt'] = \
+            tasks.TaskWorkerPool('Fault-Mgmt', num_workers=1)
+        nfvi_fault_mgmt_initialize(config, _task_worker_pools['fault_mgmt'])
+
     return init_complete
 
 
@@ -122,6 +135,7 @@ def nfvi_finalize():
     nfvi_identity_finalize()
     nfvi_guest_finalize()
     nfvi_sw_mgmt_finalize()
+    nfvi_fault_mgmt_finalize()
 
     for pool in _task_worker_pools.values():
         pool.shutdown()

nfv/nfv-vim/nfv_vim/nfvi/api/v1/__init__.py

@@ -5,6 +5,7 @@
 #
 from nfv_vim.nfvi.api.v1._nfvi_block_storage_api import NFVIBlockStorageAPI  # noqa: F401
 from nfv_vim.nfvi.api.v1._nfvi_compute_api import NFVIComputeAPI  # noqa: F401
+from nfv_vim.nfvi.api.v1._nfvi_fault_mgmt_api import NFVIFaultMgmtAPI  # noqa: F401
 from nfv_vim.nfvi.api.v1._nfvi_guest_api import NFVIGuestAPI  # noqa: F401
 from nfv_vim.nfvi.api.v1._nfvi_identity_api import NFVIIdentityAPI  # noqa: F401
 from nfv_vim.nfvi.api.v1._nfvi_image_api import NFVIImageAPI  # noqa: F401

nfv/nfv-vim/nfv_vim/nfvi/api/v1/_nfvi_fault_mgmt_api.py

@@ -0,0 +1,76 @@
+#
+# Copyright (C) 2019 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class NFVIFaultMgmtAPI(object):
+    """
+    Abstract NFVI Fault Management API Class Definition
+    """
+    @abc.abstractproperty
+    def name(self):
+        """
+        Returns the name of plugin
+        """
+        pass
+
+    @abc.abstractproperty
+    def version(self):
+        """
+        Returns the version of the plugin
+        """
+        pass
+
+    @abc.abstractproperty
+    def provider(self):
+        """
+        Returns the vendor who created the plugin
+        """
+        pass
+
+    @abc.abstractproperty
+    def signature(self):
+        """
+        Returns the signature of the plugin
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_openstack_alarms(self, future, callback):
+        """
+        Get openstack alarms using the plugin
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_openstack_logs(self, future, start_period, end_period, callback):
+        """
+        Get openstack logs using the plugin
+        """
+        pass
+
+    @abc.abstractmethod
+    def get_openstack_alarm_history(self, future, start_period, end_period, callback):
+        """
+        Get openstack alarm history using the plugin
+        """
+        pass
+
+    @abc.abstractmethod
+    def initialize(self, config_file):
+        """
+        Initialize the plugin
+        """
+        pass
+
+    @abc.abstractmethod
+    def finalize(self):
+        """
+        Finalize the plugin
+        """
+        pass

nfv/nfv-vim/nfv_vim/objects/_sw_patch.py

@@ -194,7 +194,10 @@ class SwPatch(SwUpdate):
             timer_id = (yield)
 
             DLOG.info("Audit alarms, timer_id=%s." % timer_id)
+            self.nfvi_alarms_clear()
             nfvi.nfvi_get_alarms(self.nfvi_alarms_callback(timer_id))
+            if not nfvi.nfvi_fault_mgmt_plugin_disabled():
+                nfvi.nfvi_get_openstack_alarms(self.nfvi_alarms_callback(timer_id))
             self._nfvi_audit_inprogress = True
             while self._nfvi_audit_inprogress:
                 timer_id = (yield)

nfv/nfv-vim/nfv_vim/objects/_sw_update.py

@@ -328,6 +328,9 @@ class SwUpdate(ObjectData):
         """
         raise NotImplementedError()
 
+    def nfvi_alarms_clear(self):
+        self._nfvi_alarms = list()
+
     @coroutine
     def nfvi_alarms_callback(self, timer_id):
         """
@@ -338,7 +341,7 @@ class SwUpdate(ObjectData):
         if response['completed']:
             DLOG.verbose("Audit-Alarms callback, response=%s." % response)
 
-            self._nfvi_alarms = response['result-data']
+            self._nfvi_alarms.extend(response['result-data'])
         else:
             DLOG.error("Audit-Alarms callback, not completed, "
                        "response=%s." % response)

nfv/nfv-vim/nfv_vim/objects/_sw_upgrade.py

@@ -154,7 +154,10 @@ class SwUpgrade(SwUpdate):
             timer_id = (yield)
 
             DLOG.info("Audit alarms, timer_id=%s." % timer_id)
+            self.nfvi_alarms_clear()
             nfvi.nfvi_get_alarms(self.nfvi_alarms_callback(timer_id))
+            if not nfvi.nfvi_fault_mgmt_plugin_disabled():
+                nfvi.nfvi_get_openstack_alarms(self.nfvi_alarms_callback(timer_id))
             self._nfvi_audit_inprogress = True
             while self._nfvi_audit_inprogress:
                 timer_id = (yield)

nfv/nfv-vim/nfv_vim/strategy/_strategy_steps.py

@@ -1471,7 +1471,7 @@ class QueryAlarmsStep(strategy.StrategyStep):
         self._ignore_alarms = ignore_alarms
 
     @coroutine
-    def _query_alarms_callback(self):
+    def _query_alarms_callback(self, fm_service):
         """
         Query Alarms Callback
         """
@@ -1480,7 +1480,7 @@ class QueryAlarmsStep(strategy.StrategyStep):
 
         if response['completed']:
             if self.strategy is not None:
-                nfvi_alarms = list()
+                nfvi_alarms = self.strategy.nfvi_alarms
                 for nfvi_alarm in response['result-data']:
                     if (self.strategy._alarm_restrictions ==
                             strategy.STRATEGY_ALARM_RESTRICTION_TYPES.RELAXED and
@@ -1498,7 +1498,7 @@ class QueryAlarmsStep(strategy.StrategyStep):
 
             if self._fail_on_alarms and self.strategy.nfvi_alarms:
                 result = strategy.STRATEGY_STEP_RESULT.FAILED
-                reason = "alarms are present"
+                reason = "alarms from %s are present" % fm_service
             else:
                 result = strategy.STRATEGY_STEP_RESULT.SUCCESS
                 reason = ""
@@ -1515,7 +1515,10 @@ class QueryAlarmsStep(strategy.StrategyStep):
         from nfv_vim import nfvi
 
         DLOG.info("Step (%s) apply." % self._name)
-        nfvi.nfvi_get_alarms(self._query_alarms_callback())
+        self.strategy.nfvi_alarms = list()
+        nfvi.nfvi_get_alarms(self._query_alarms_callback("platform"))
+        if not nfvi.nfvi_fault_mgmt_plugin_disabled():
+            nfvi.nfvi_get_openstack_alarms(self._query_alarms_callback("openstack"))
         return strategy.STRATEGY_STEP_RESULT.WAIT, ""
 
     def from_dict(self, data):