diff --git a/devstack/plugin.sh b/devstack/plugin.sh index bc84953cb..02a12aa9a 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -189,6 +189,11 @@ function configure_vitrage { disable_vitrage_datasource trove.instance trove.cluster fi + # remove monasca vitrage datasource if nagios datasource not installed + if ! is_service_enabled monasca; then + disable_vitrage_datasource monasca + fi + # remove nagios vitrage datasource if nagios datasource not installed if [[ "$VITRAGE_USE_NAGIOS" == "False" ]]; then disable_vitrage_datasource nagios @@ -342,6 +347,12 @@ function modify_heat_global_index_policy_rule { fi } +function add_monasca_rule_to_vitrage { + if is_service_enabled monasca; then + get_or_add_user_project_role "monasca-read-only-user" "vitrage" "admin" + fi +} + # This is the main for plugin.sh if is_service_enabled vitrage; then if [[ "$1" == "stack" && "$2" == "pre-install" ]]; then @@ -361,6 +372,9 @@ if is_service_enabled vitrage; then modify_heat_global_index_policy_rule # Tidy base for vitrage init_vitrage + # add monasca-read-only-user role to vitrage user so + # vitrage user can read monasca alarms + add_monasca_rule_to_vitrage # Start the services start_vitrage fi diff --git a/devstack/settings b/devstack/settings index 1435de686..31e06ed7b 100644 --- a/devstack/settings +++ b/devstack/settings @@ -36,7 +36,7 @@ VITRAGE_USE_STATIC=$(trueorfalse False VITRAGE_USE_STATIC) VITRAGE_USE_DOCTOR=$(trueorfalse False VITRAGE_USE_DOCTOR) VITRAGE_USE_PROMETHEUS=$(trueorfalse False VITRAGE_USE_PROMETHEUS) -VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus,trove.instance,trove.cluster} +VITRAGE_DEFAULT_DATASOURCES=${VITRAGE_DEFAULT_DATASOURCES:-nova.host,nova.instance,nova.zone,nagios,static,aodh,cinder.volume,neutron.network,neutron.port,heat.stack,doctor,prometheus,trove.instance,trove.cluster,monasca} # for now dont use pip install for the client LIBS_FROM_GIT=python-vitrageclient diff --git a/doc/requirements.txt b/doc/requirements.txt index 93c373c1c..9b5e73810 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -3,6 +3,7 @@ # process, which may cause wedges in the gate later. oslo.config>=5.2.0 # Apache-2.0 openstackdocstheme>=1.24.0 # Apache-2.0 -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2,<2.0.0;python_version=='2.7' # BSD +sphinx!=1.6.6,!=1.6.7,>=1.6.2;python_version>='3.4' # BSD reno>=2.7.0 # Apache-2.0 docutils>=0.11 # OSI-Approved Open Source, Public Domain diff --git a/etc/vitrage/datasources_values/monasca.yaml b/etc/vitrage/datasources_values/monasca.yaml new file mode 100644 index 000000000..7a5bc00d9 --- /dev/null +++ b/etc/vitrage/datasources_values/monasca.yaml @@ -0,0 +1,17 @@ +category: ALARM +values: + - aggregated values: + priority: 30 + original values: + - name: ALARM + operational_value: CRITICAL + - aggregated values: + priority: 20 + original values: + - name: UNDETERMINED + operational_value: N/A + - aggregated values: + priority: 10 + original values: + - name: OK + operational_value: OK diff --git a/lower-constraints.txt b/lower-constraints.txt index f71395042..b53dfe6b9 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -50,7 +50,8 @@ msgpack==0.5.6 munch==2.2.0 netaddr==0.7.19 netifaces==0.10.6 -networkx==2.0 +networkx==2.3;python_version>='3.4' +networkx==2.0;python_version<'3.0' openstacksdk==0.12.0 os-client-config==1.29.0 os-service-types==1.2.0 @@ -98,6 +99,7 @@ python-heatclient==1.14.0 python-keystoneclient==3.15.0 python-mimeparse==1.6.0 python-mistralclient==3.3.0 +python-monascaclient==1.7.1 python-neutronclient==6.7.0 python-novaclient==10.1.0 python-openstackclient==3.12.0 diff --git a/releasenotes/notes/monasca-datasource-9ca61922ef14c2a8.yaml b/releasenotes/notes/monasca-datasource-9ca61922ef14c2a8.yaml new file mode 100644 index 000000000..227062e4c --- /dev/null +++ b/releasenotes/notes/monasca-datasource-9ca61922ef14c2a8.yaml @@ -0,0 +1,20 @@ +--- +features: + - A new ``Monasca Datasource`` has been introduced to include Monasca alarms + in Vitrage Entity Graph. Monasca is Monitoring as a Service solution + offering centralized monitoring sink for metrics gathered by Monasca Agents + at many infrastructure levels. Moreover it provides alarm management API + that enables defining alarms based on collected metrics. + + This change is the first stage of integration with Monasca. At this point, + Monasca entities are extracted using PULL approach, based on periodical + snapshot-query to Monasca Alarm API for the current list of alarm entities. + In the future, PUSH approach based on Monasca notifications will be + implemented. + + Current implementation requires that the metrics associated with the given + alarm contain information about resource type and ID - required for + associating alarms with entities in Vitrage Entity Graph. This additional + information should be included in the form of metric dimensions, precisely + ``resource_type`` and ``resource_id``. Dimensions can be defined in Monasca + agent configuration. diff --git a/requirements.txt b/requirements.txt index df41497bb..a1de30f30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,11 +15,13 @@ python-novaclient>=10.1.0 # Apache-2.0 python-heatclient>=1.14.0 # Apache-2.0 python-mistralclient>=3.3.0 # Apache-2.0 python-openstackclient>=3.12.0 # Apache-2.0 +python-monascaclient>=1.7.1 # Apache-2.0 python-troveclient>=2.2.0 # Apache-2.0 python-zaqarclient >=1.2.0 gnocchiclient>=3.3.1 # Apache-2.0 pyzabbix>=0.7.4 # LGPL -networkx>=2.0 # BSD +networkx<2.3,>=2.0;python_version<'3.0' # BSD +networkx>=2.3;python_version>='3.4' # BSD oslo.config>=5.2.0 # Apache-2.0 oslo.context>=2.20.0 # Apache-2.0 oslo.db>=4.35.0 # Apache-2.0 diff --git a/vitrage/datasources/monasca/__init__.py b/vitrage/datasources/monasca/__init__.py new file mode 100644 index 000000000..4a4cd0680 --- /dev/null +++ b/vitrage/datasources/monasca/__init__.py @@ -0,0 +1,45 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from vitrage.common.constants import DatasourceOpts as DSOpts +from vitrage.common.constants import UpdateMethod + +MONASCA_DATASOURCE = 'monasca' + +OPTS = [ + cfg.StrOpt(DSOpts.TRANSFORMER, + default='vitrage.datasources.monasca.transformer.' + 'MonascaTransformer', + help='Monasca transformer class path.', + required=True), + cfg.StrOpt(DSOpts.DRIVER, + default='vitrage.datasources.monasca.driver.' + 'MonascaDriver', + help='Monasca driver class path.', + required=True), + cfg.StrOpt(DSOpts.UPDATE_METHOD, + default=UpdateMethod.PULL, + help='None: updates only via Vitrage periodic snapshots.' + 'Pull: updates periodically.' + 'Push: updates by getting notifications from the' + ' datasource itself.', + required=True), + cfg.IntOpt(DSOpts.CHANGES_INTERVAL, + default=30, + min=10, + help='Interval in seconds between checking changes in Monasca' + 'datasource.'), + ] diff --git a/vitrage/datasources/monasca/driver.py b/vitrage/datasources/monasca/driver.py new file mode 100644 index 000000000..420213a0c --- /dev/null +++ b/vitrage/datasources/monasca/driver.py @@ -0,0 +1,76 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log + +from vitrage.datasources.alarm_driver_base import AlarmDriverBase +from vitrage.datasources.monasca import MONASCA_DATASOURCE +from vitrage.datasources.monasca.properties import ( + MonascaAlarmStatuses as MAlarmStatuses) +from vitrage.datasources.monasca.properties import MonascaProperties as MProps +from vitrage.datasources.transformer_base import extract_field_value +from vitrage import os_clients + +LOG = log.getLogger(__name__) + + +class MonascaDriver(AlarmDriverBase): + + def __init__(self, conf): + super(MonascaDriver, self).__init__() + self.conf = conf + self.__client = None + + @property + def client(self): + if not self.__client: + self.__client = os_clients.monasca_client(self.conf) + return self.__client + + def _vitrage_type(self): + return MONASCA_DATASOURCE + + def _alarm_key(self, alarm): + return alarm[MProps.ID] + + def _get_alarms(self): + try: + return self.client.alarms.list() + except Exception: + LOG.exception("Failed to fetch Monasca alarms.") + return [] + + def _enrich_alarms(self, alarms): + for alarm in alarms: + alarm[MProps.RESOURCE_TYPE] = extract_field_value( + alarm, 'metrics', 0, 'dimensions', 'resource_type') + alarm[MProps.RESOURCE_ID] = extract_field_value( + alarm, 'metrics', 0, 'dimensions', 'resource_id') + + def _is_erroneous(self, alarm): + return alarm and alarm[MProps.STATUS] == MAlarmStatuses.ALARM + + def _status_changed(self, new_alarm, old_alarm): + return new_alarm and old_alarm and \ + new_alarm[MProps.STATUS] != old_alarm[MProps.STATUS] + + def _is_valid(self, alarm): + return alarm and \ + alarm[MProps.RESOURCE_TYPE] is not None and \ + alarm[MProps.RESOURCE_ID] is not None + + @staticmethod + def should_delete_outdated_entities(): + return True diff --git a/vitrage/datasources/monasca/properties.py b/vitrage/datasources/monasca/properties.py new file mode 100644 index 000000000..1f7dc6c9d --- /dev/null +++ b/vitrage/datasources/monasca/properties.py @@ -0,0 +1,30 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class MonascaProperties(object): + + ID = 'id' + NAME = 'name' + RESOURCE_TYPE = 'resource_type' + RESOURCE_ID = 'resource_id' + STATUS = 'state' + UPDATE_TIMESTAMP = 'updated_timestamp' + + +class MonascaAlarmStatuses(object): + + OK = 'OK' + ALARM = 'ALARM' diff --git a/vitrage/datasources/monasca/transformer.py b/vitrage/datasources/monasca/transformer.py new file mode 100644 index 000000000..dc359e98b --- /dev/null +++ b/vitrage/datasources/monasca/transformer.py @@ -0,0 +1,85 @@ +# Copyright 2018 Samsung Electronics +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EdgeLabel +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.alarm_transformer_base import AlarmTransformerBase +from vitrage.datasources.monasca import MONASCA_DATASOURCE +from vitrage.datasources.monasca.properties import ( + MonascaAlarmStatuses as MAlarmStatuses) +from vitrage.datasources.monasca.properties import MonascaProperties as MProps +from vitrage.datasources import transformer_base as tbase +from vitrage.datasources.transformer_base import extract_field_value +import vitrage.graph.utils as graph_utils + + +class MonascaTransformer(AlarmTransformerBase): + + def _create_snapshot_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_update_entity_vertex(self, entity_event): + return self._create_vertex(entity_event) + + def _create_vertex(self, entity_event): + update_timestamp = entity_event[MProps.UPDATE_TIMESTAMP] + sample_timestamp = entity_event[DSProps.SAMPLE_DATE] + + name = extract_field_value( + entity_event, 'alarm_definition', 'name') + + metadata = { + VProps.NAME: name, + VProps.VITRAGE_RESOURCE_ID: entity_event[MProps.RESOURCE_ID], + VProps.VITRAGE_RESOURCE_TYPE: entity_event[MProps.RESOURCE_TYPE], + VProps.SEVERITY: entity_event[MProps.STATUS] + } + + return graph_utils.create_vertex( + self._create_entity_key(entity_event), + vitrage_category=EntityCategory.ALARM, + vitrage_type=MONASCA_DATASOURCE, + vitrage_sample_timestamp=sample_timestamp, + entity_id=entity_event[MProps.ID], + entity_state=self._get_alarm_state(entity_event), + update_timestamp=update_timestamp, + metadata=metadata) + + def _create_snapshot_neighbors(self, entity_event): + return self._create_monasca_neighbors(entity_event) + + def _create_update_neighbors(self, entity_event): + return self._create_monasca_neighbors(entity_event) + + def _create_monasca_neighbors(self, entity_event): + return [self._create_neighbor( + entity_event, + entity_event[MProps.RESOURCE_ID], + entity_event[MProps.RESOURCE_TYPE], + EdgeLabel.ON, + neighbor_category=EntityCategory.RESOURCE)] + + def _ok_status(self, entity_event): + return entity_event[MProps.STATUS] == MAlarmStatuses.OK + + def _create_entity_key(self, entity_event): + entity_id = entity_event[MProps.ID] + key_fields = self._key_values(MONASCA_DATASOURCE, entity_id) + return tbase.build_key(key_fields) + + def get_vitrage_type(self): + return MONASCA_DATASOURCE diff --git a/vitrage/os_clients.py b/vitrage/os_clients.py index 724a4a07d..2f3d74f0b 100644 --- a/vitrage/os_clients.py +++ b/vitrage/os_clients.py @@ -18,6 +18,7 @@ from oslo_utils import importutils as utils from vitrage import keystone_client + LOG = log.getLogger(__name__) OPTS = [ @@ -30,6 +31,7 @@ OPTS = [ cfg.StrOpt('mistral_version', default='2', help='Mistral version'), cfg.StrOpt('gnocchi_version', default='1', help='Gnocchi version'), cfg.StrOpt('trove_version', default='1', help='Trove version'), + cfg.StrOpt('monasca_version', default='2_0', help='Monasca version'), cfg.BoolOpt('use_nova_versioned_notifications', default=True, help='Indicates whether to use Nova versioned notifications.' @@ -49,7 +51,8 @@ _client_modules = { 'heat': 'heatclient.client', 'mistral': 'mistralclient.api.v2.client', 'gnocchi': 'gnocchiclient.v1.client', - 'trove': 'troveclient.v1.client' + 'trove': 'troveclient.v1.client', + 'monasca': 'monascaclient.client' } @@ -205,3 +208,22 @@ def zaqar_client(conf): return client except Exception: LOG.exception('Create Zaqar client - Got Exception.') + + +def monasca_client(conf): + """Get an instance of Monasca client""" + try: + mon_client = driver_module('monasca') + + session = keystone_client.get_session(conf) + endpoint = session.get_endpoint(service_type='monitoring', + interface='publicURL') + client = mon_client.Client( + api_version=conf.monasca_version, + session=session, + endpoint=endpoint + ) + LOG.info('Monasca client created') + return client + except Exception: + LOG.exception('Create Monasca client - Got Exception.')