From 1c49907adaca3af3b58ce5752daf9fdae30a0422 Mon Sep 17 00:00:00 2001 From: Ifat Afek Date: Mon, 10 Jul 2017 16:49:46 +0000 Subject: [PATCH] Support different resource types in collectd Currently, due to a bug, collectd alarms are supported only on hosts. I fixed the bug + added tests Change-Id: Ie2760689e5cd248f6d499f2815e86d36c011dc6e --- vitrage/common/constants.py | 1 + vitrage/datasources/collectd/transformer.py | 34 +-- .../datasources/collectd/__init__.py | 15 ++ .../datasources/collectd/test_collectd.py | 200 ++++++++++++++++++ .../collectd/test_collectd_transformer.py | 10 +- 5 files changed, 246 insertions(+), 14 deletions(-) create mode 100644 vitrage/tests/functional/datasources/collectd/__init__.py create mode 100644 vitrage/tests/functional/datasources/collectd/test_collectd.py diff --git a/vitrage/common/constants.py b/vitrage/common/constants.py index 3c05ef37c..c84e396ab 100644 --- a/vitrage/common/constants.py +++ b/vitrage/common/constants.py @@ -38,6 +38,7 @@ class VertexProperties(object): GRAPH_INDEX = 'graph_index' RAWTEXT = 'rawtext' RESOURCE_ID = 'resource_id' + RESOURCE_NAME = 'resource_name' VITRAGE_RESOURCE_ID = 'vitrage_resource_id' VITRAGE_RESOURCE_TYPE = 'vitrage_resource_type' RESOURCE = 'resource' diff --git a/vitrage/datasources/collectd/transformer.py b/vitrage/datasources/collectd/transformer.py index bfebaf2f8..549d86723 100644 --- a/vitrage/datasources/collectd/transformer.py +++ b/vitrage/datasources/collectd/transformer.py @@ -14,7 +14,7 @@ from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import EdgeLabel -from vitrage.common.constants import EntityCategory +from vitrage.common.constants import EntityCategory as Category from vitrage.common.constants import VertexProperties as VProps from vitrage.datasources.alarm_properties import AlarmProperties as AlarmProps from vitrage.datasources.alarm_transformer_base import AlarmTransformerBase @@ -54,12 +54,12 @@ class CollectdTransformer(AlarmTransformerBase): VProps.NAME: entity_event[CProps.MESSAGE], VProps.SEVERITY: entity_event[CProps.SEVERITY], VProps.RAWTEXT: self.generate_raw_text(entity_event), - VProps.RESOURCE_ID: entity_event[CProps.RESOURCE_NAME] + VProps.RESOURCE_NAME: entity_event[CProps.RESOURCE_NAME] } return graph_utils.create_vertex( self._create_entity_key(entity_event), - vitrage_category=EntityCategory.ALARM, + vitrage_category=Category.ALARM, vitrage_type=entity_event[DSProps.ENTITY_TYPE], vitrage_sample_timestamp=vitrage_sample_timestamp, entity_state=entity_state, @@ -73,17 +73,14 @@ class CollectdTransformer(AlarmTransformerBase): return self._create_collectd_neighbors(entity_event) def _create_collectd_neighbors(self, entity_event): + graph_neighbors = entity_event.get(self.QUERY_RESULT, []) - resource_type = entity_event[CProps.RESOURCE_TYPE] - if resource_type: - return [self._create_neighbor( - entity_event, - entity_event[CProps.RESOURCE_NAME], - resource_type, - EdgeLabel.ON, - neighbor_category=EntityCategory.RESOURCE)] - - return [] + return [self._create_neighbor(entity_event, + graph_neighbor[VProps.ID], + graph_neighbor[VProps.VITRAGE_TYPE], + EdgeLabel.ON, + neighbor_category=Category.RESOURCE) + for graph_neighbor in graph_neighbors] def _ok_status(self, entity_event): return entity_event[CProps.SEVERITY] == 'OK' @@ -106,3 +103,14 @@ class CollectdTransformer(AlarmTransformerBase): entity_event[CProps.PLUGIN], entity_event.get(CProps.PLUGIN_INSTANCE)] return '-'.join([resource for resource in resources if resource]) + + @staticmethod + def get_enrich_query(event): + resource_type = event.get(CProps.RESOURCE_TYPE) + resource_name = event.get(CProps.RESOURCE_NAME) + + if resource_type and resource_name: + return {VProps.NAME: resource_name, + VProps.VITRAGE_TYPE: resource_type} + + return None diff --git a/vitrage/tests/functional/datasources/collectd/__init__.py b/vitrage/tests/functional/datasources/collectd/__init__.py new file mode 100644 index 000000000..bf9f61d74 --- /dev/null +++ b/vitrage/tests/functional/datasources/collectd/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/tests/functional/datasources/collectd/test_collectd.py b/vitrage/tests/functional/datasources/collectd/test_collectd.py new file mode 100644 index 000000000..3955e9fe9 --- /dev/null +++ b/vitrage/tests/functional/datasources/collectd/test_collectd.py @@ -0,0 +1,200 @@ +# Copyright 2017 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import time + +from oslo_config import cfg + +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.common.constants import EntityCategory +from vitrage.common.constants import VertexProperties as VProps +from vitrage.datasources.collectd import COLLECTD_DATASOURCE +from vitrage.datasources.collectd.properties import \ + CollectdProperties as CProps +from vitrage.datasources import NOVA_HOST_DATASOURCE +from vitrage.datasources import NOVA_INSTANCE_DATASOURCE +from vitrage.datasources import NOVA_ZONE_DATASOURCE +from vitrage.tests.functional.datasources.base import \ + TestDataSourcesBase +from vitrage.tests.mocks import mock_transformer +from vitrage.utils.datetime import format_unix_timestamp + + +class TestCollectd(TestDataSourcesBase): + + DATASOURCES_OPTS = [ + cfg.ListOpt('types', + default=[COLLECTD_DATASOURCE, + NOVA_HOST_DATASOURCE, + NOVA_INSTANCE_DATASOURCE, + NOVA_ZONE_DATASOURCE], + help='Names of supported driver data sources'), + + cfg.ListOpt('path', + default=['vitrage.datasources'], + help='base path for data sources') + ] + + # noinspection PyPep8Naming + @classmethod + def setUpClass(cls): + super(TestCollectd, cls).setUpClass() + cls.conf = cfg.ConfigOpts() + cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph') + cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources') + cls.load_datasources(cls.conf) + + def test_collectd_alarm_on_host(self): + self._test_collectd_alarm(NOVA_HOST_DATASOURCE, 'host-2', 'host-2') + + def test_collectd_alarm_on_instance(self): + self._test_collectd_alarm(NOVA_INSTANCE_DATASOURCE, 'vm-5', 'host-4') + + def _test_collectd_alarm(self, resource_type, resource_name, host_name): + # Setup + processor = self._create_processor_with_graph(self.conf, uuid=True) + self.assertEqual(self._num_total_expected_vertices(), + len(processor.entity_graph)) + + time1 = time.time() + severity1 = 'WARNING' + collectd_event = self._create_collectd_event(time1, + resource_type, + resource_name, + host_name, + severity1) + + # Action + processor.process_event(collectd_event) + + # Test assertions + self.assertEqual(self._num_total_expected_vertices() + 1, + len(processor.entity_graph)) + + collectd_vertices = processor.entity_graph.get_vertices( + vertex_attr_filter={ + VProps.VITRAGE_CATEGORY: EntityCategory.ALARM, + VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE + }) + + self.assertEqual(1, len(collectd_vertices)) + collectd_vertex1 = collectd_vertices[0] + self._assert_collectd_vertex_equals(collectd_vertex1, + time1, + resource_type, + resource_name, + severity1) + + collectd_neighbors = processor.entity_graph.neighbors( + collectd_vertices[0].vertex_id) + + self._assert_collectd_neighbor_equals(collectd_neighbors, + resource_type, + resource_name) + + # Action 2 - update the existing alarm + time2 = time.time() + severity2 = 'ERROR' + collectd_event = self._create_collectd_event(time2, + resource_type, + resource_name, + host_name, + severity2) + + processor.process_event(collectd_event) + + # Test assertions - the collectd alarm vertex should be the same + self.assertEqual(self._num_total_expected_vertices() + 1, + len(processor.entity_graph)) + + collectd_vertices = processor.entity_graph.get_vertices( + vertex_attr_filter={ + VProps.VITRAGE_CATEGORY: EntityCategory.ALARM, + VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE + }) + + self.assertEqual(1, len(collectd_vertices)) + collectd_vertex2 = collectd_vertices[0] + self.assertEqual(collectd_vertex1[VProps.VITRAGE_ID], + collectd_vertex2[VProps.VITRAGE_ID]) + + # Action 3 - clear the alarm + time3 = time.time() + severity3 = 'OK' + collectd_event = self._create_collectd_event(time3, + resource_type, + resource_name, + host_name, + severity3) + + processor.process_event(collectd_event) + + # Test assertions - the collectd alarm vertex should be removed + collectd_vertices = processor.entity_graph.get_vertices( + vertex_attr_filter={ + VProps.VITRAGE_CATEGORY: EntityCategory.ALARM, + VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE + }) + + self._assert_no_vertex(collectd_vertices) + + @staticmethod + def _create_collectd_event(time, + resource_type, + resource_name, + host_name, + severity): + update_vals = {CProps.TIME: time, + DSProps.SAMPLE_DATE: format_unix_timestamp(time), + CProps.HOST: host_name, + CProps.RESOURCE_TYPE: resource_type, + CProps.RESOURCE_NAME: resource_name, + CProps.MESSAGE: 'A message for you', + CProps.SEVERITY: severity} + + spec_list = mock_transformer.simple_collectd_alarm_generators( + update_vals=update_vals) + static_events = mock_transformer.generate_random_events_list(spec_list) + + return static_events[0] + + def _assert_collectd_vertex_equals(self, + collectd_vertex, + expected_time, + expected_resource_type, + expected_resource_name, + expected_severity): + + self.assertEqual(format_unix_timestamp(expected_time), + collectd_vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP]) + self.assertEqual(expected_resource_type, + collectd_vertex[VProps.VITRAGE_RESOURCE_TYPE]) + self.assertEqual(expected_resource_name, + collectd_vertex[CProps.RESOURCE_NAME]) + self.assertEqual(expected_severity, collectd_vertex[VProps.SEVERITY]) + + def _assert_collectd_neighbor_equals(self, + collectd_neighbors, + expected_resource_type, + expected_resource_name): + self.assertEqual(1, len(collectd_neighbors)) + + self.assertEqual(expected_resource_type, + collectd_neighbors[0][VProps.VITRAGE_TYPE]) + self.assertEqual(expected_resource_name, + collectd_neighbors[0][VProps.NAME]) + + def _assert_no_vertex(self, vertices): + self.assertTrue(len(vertices) == 0 or + (len(vertices) == 1 and + vertices[0].get(VProps.VITRAGE_IS_DELETED, False))) diff --git a/vitrage/tests/unit/datasources/collectd/test_collectd_transformer.py b/vitrage/tests/unit/datasources/collectd/test_collectd_transformer.py index e97c69d3e..b9525867d 100644 --- a/vitrage/tests/unit/datasources/collectd/test_collectd_transformer.py +++ b/vitrage/tests/unit/datasources/collectd/test_collectd_transformer.py @@ -19,12 +19,14 @@ from oslo_config import cfg from vitrage.common.constants import DatasourceOpts as DSOpts from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import UpdateMethod +from vitrage.common.constants import VertexProperties as VProps from vitrage.datasources.collectd import COLLECTD_DATASOURCE from vitrage.datasources.collectd.properties import \ CollectdProperties as CProps from vitrage.datasources.collectd.transformer import CollectdTransformer from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.nova.host.transformer import HostTransformer +from vitrage.datasources.transformer_base import TransformerBase from vitrage.tests.mocks import mock_transformer from vitrage.tests.unit.datasources.test_alarm_transformer_base import \ BaseAlarmTransformerTest @@ -99,11 +101,17 @@ class TestCollectdTransformer(BaseAlarmTransformerTest): @staticmethod def _generate_event(time, hostname, severity): + # fake query result to be used by the transformer for determining + # the neighbor + query_result = [{VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE, + VProps.ID: hostname}] + update_vals = {CProps.HOST: hostname, CProps.SEVERITY: severity, CProps.TIME: time, DSProps.SAMPLE_DATE: format_unix_timestamp(time), - CProps.RESOURCE_NAME: hostname} + CProps.RESOURCE_NAME: hostname, + TransformerBase.QUERY_RESULT: query_result} generators = mock_transformer.simple_collectd_alarm_generators( update_vals=update_vals)