Support different resource types in collectd

Currently, due to a bug, collectd alarms are supported only on hosts. I fixed the bug + added tests

Change-Id: Ie2760689e5cd248f6d499f2815e86d36c011dc6e

vitrage/common/constants.py

@@ -38,6 +38,7 @@ class VertexProperties(object):
     GRAPH_INDEX = 'graph_index'
     RAWTEXT = 'rawtext'
     RESOURCE_ID = 'resource_id'
+    RESOURCE_NAME = 'resource_name'
     VITRAGE_RESOURCE_ID = 'vitrage_resource_id'
     VITRAGE_RESOURCE_TYPE = 'vitrage_resource_type'
     RESOURCE = 'resource'

vitrage/datasources/collectd/transformer.py

@@ -14,7 +14,7 @@
 
 from vitrage.common.constants import DatasourceProperties as DSProps
 from vitrage.common.constants import EdgeLabel
-from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import EntityCategory as Category
 from vitrage.common.constants import VertexProperties as VProps
 from vitrage.datasources.alarm_properties import AlarmProperties as AlarmProps
 from vitrage.datasources.alarm_transformer_base import AlarmTransformerBase
@@ -54,12 +54,12 @@ class CollectdTransformer(AlarmTransformerBase):
             VProps.NAME: entity_event[CProps.MESSAGE],
             VProps.SEVERITY: entity_event[CProps.SEVERITY],
             VProps.RAWTEXT: self.generate_raw_text(entity_event),
-            VProps.RESOURCE_ID: entity_event[CProps.RESOURCE_NAME]
+            VProps.RESOURCE_NAME: entity_event[CProps.RESOURCE_NAME]
         }
 
         return graph_utils.create_vertex(
             self._create_entity_key(entity_event),
-            vitrage_category=EntityCategory.ALARM,
+            vitrage_category=Category.ALARM,
             vitrage_type=entity_event[DSProps.ENTITY_TYPE],
             vitrage_sample_timestamp=vitrage_sample_timestamp,
             entity_state=entity_state,
@@ -73,17 +73,14 @@ class CollectdTransformer(AlarmTransformerBase):
         return self._create_collectd_neighbors(entity_event)
 
     def _create_collectd_neighbors(self, entity_event):
+        graph_neighbors = entity_event.get(self.QUERY_RESULT, [])
 
-        resource_type = entity_event[CProps.RESOURCE_TYPE]
+        return [self._create_neighbor(entity_event,
-        if resource_type:
+                                      graph_neighbor[VProps.ID],
-            return [self._create_neighbor(
+                                      graph_neighbor[VProps.VITRAGE_TYPE],
-                entity_event,
+                                      EdgeLabel.ON,
-                entity_event[CProps.RESOURCE_NAME],
+                                      neighbor_category=Category.RESOURCE)
-                resource_type,
+                for graph_neighbor in graph_neighbors]
-                EdgeLabel.ON,
-                neighbor_category=EntityCategory.RESOURCE)]
-
-        return []
 
     def _ok_status(self, entity_event):
         return entity_event[CProps.SEVERITY] == 'OK'
@@ -106,3 +103,14 @@ class CollectdTransformer(AlarmTransformerBase):
                      entity_event[CProps.PLUGIN],
                      entity_event.get(CProps.PLUGIN_INSTANCE)]
         return '-'.join([resource for resource in resources if resource])
+
+    @staticmethod
+    def get_enrich_query(event):
+        resource_type = event.get(CProps.RESOURCE_TYPE)
+        resource_name = event.get(CProps.RESOURCE_NAME)
+
+        if resource_type and resource_name:
+            return {VProps.NAME: resource_name,
+                    VProps.VITRAGE_TYPE: resource_type}
+
+        return None

vitrage/tests/functional/datasources/collectd/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2017 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+__author__ = 'stack'

vitrage/tests/functional/datasources/collectd/test_collectd.py

@@ -0,0 +1,200 @@
+# Copyright 2017 - Nokia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+import time
+
+from oslo_config import cfg
+
+from vitrage.common.constants import DatasourceProperties as DSProps
+from vitrage.common.constants import EntityCategory
+from vitrage.common.constants import VertexProperties as VProps
+from vitrage.datasources.collectd import COLLECTD_DATASOURCE
+from vitrage.datasources.collectd.properties import \
+    CollectdProperties as CProps
+from vitrage.datasources import NOVA_HOST_DATASOURCE
+from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
+from vitrage.datasources import NOVA_ZONE_DATASOURCE
+from vitrage.tests.functional.datasources.base import \
+    TestDataSourcesBase
+from vitrage.tests.mocks import mock_transformer
+from vitrage.utils.datetime import format_unix_timestamp
+
+
+class TestCollectd(TestDataSourcesBase):
+
+    DATASOURCES_OPTS = [
+        cfg.ListOpt('types',
+                    default=[COLLECTD_DATASOURCE,
+                             NOVA_HOST_DATASOURCE,
+                             NOVA_INSTANCE_DATASOURCE,
+                             NOVA_ZONE_DATASOURCE],
+                    help='Names of supported driver data sources'),
+
+        cfg.ListOpt('path',
+                    default=['vitrage.datasources'],
+                    help='base path for data sources')
+    ]
+
+    # noinspection PyPep8Naming
+    @classmethod
+    def setUpClass(cls):
+        super(TestCollectd, cls).setUpClass()
+        cls.conf = cfg.ConfigOpts()
+        cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
+        cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
+        cls.load_datasources(cls.conf)
+
+    def test_collectd_alarm_on_host(self):
+        self._test_collectd_alarm(NOVA_HOST_DATASOURCE, 'host-2', 'host-2')
+
+    def test_collectd_alarm_on_instance(self):
+        self._test_collectd_alarm(NOVA_INSTANCE_DATASOURCE, 'vm-5', 'host-4')
+
+    def _test_collectd_alarm(self, resource_type, resource_name, host_name):
+        # Setup
+        processor = self._create_processor_with_graph(self.conf, uuid=True)
+        self.assertEqual(self._num_total_expected_vertices(),
+                         len(processor.entity_graph))
+
+        time1 = time.time()
+        severity1 = 'WARNING'
+        collectd_event = self._create_collectd_event(time1,
+                                                     resource_type,
+                                                     resource_name,
+                                                     host_name,
+                                                     severity1)
+
+        # Action
+        processor.process_event(collectd_event)
+
+        # Test assertions
+        self.assertEqual(self._num_total_expected_vertices() + 1,
+                         len(processor.entity_graph))
+
+        collectd_vertices = processor.entity_graph.get_vertices(
+            vertex_attr_filter={
+                VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
+                VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE
+            })
+
+        self.assertEqual(1, len(collectd_vertices))
+        collectd_vertex1 = collectd_vertices[0]
+        self._assert_collectd_vertex_equals(collectd_vertex1,
+                                            time1,
+                                            resource_type,
+                                            resource_name,
+                                            severity1)
+
+        collectd_neighbors = processor.entity_graph.neighbors(
+            collectd_vertices[0].vertex_id)
+
+        self._assert_collectd_neighbor_equals(collectd_neighbors,
+                                              resource_type,
+                                              resource_name)
+
+        # Action 2 - update the existing alarm
+        time2 = time.time()
+        severity2 = 'ERROR'
+        collectd_event = self._create_collectd_event(time2,
+                                                     resource_type,
+                                                     resource_name,
+                                                     host_name,
+                                                     severity2)
+
+        processor.process_event(collectd_event)
+
+        # Test assertions - the collectd alarm vertex should be the same
+        self.assertEqual(self._num_total_expected_vertices() + 1,
+                         len(processor.entity_graph))
+
+        collectd_vertices = processor.entity_graph.get_vertices(
+            vertex_attr_filter={
+                VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
+                VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE
+            })
+
+        self.assertEqual(1, len(collectd_vertices))
+        collectd_vertex2 = collectd_vertices[0]
+        self.assertEqual(collectd_vertex1[VProps.VITRAGE_ID],
+                         collectd_vertex2[VProps.VITRAGE_ID])
+
+        # Action 3 - clear the alarm
+        time3 = time.time()
+        severity3 = 'OK'
+        collectd_event = self._create_collectd_event(time3,
+                                                     resource_type,
+                                                     resource_name,
+                                                     host_name,
+                                                     severity3)
+
+        processor.process_event(collectd_event)
+
+        # Test assertions - the collectd alarm vertex should be removed
+        collectd_vertices = processor.entity_graph.get_vertices(
+            vertex_attr_filter={
+                VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
+                VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE
+            })
+
+        self._assert_no_vertex(collectd_vertices)
+
+    @staticmethod
+    def _create_collectd_event(time,
+                               resource_type,
+                               resource_name,
+                               host_name,
+                               severity):
+        update_vals = {CProps.TIME: time,
+                       DSProps.SAMPLE_DATE: format_unix_timestamp(time),
+                       CProps.HOST: host_name,
+                       CProps.RESOURCE_TYPE: resource_type,
+                       CProps.RESOURCE_NAME: resource_name,
+                       CProps.MESSAGE: 'A message for you',
+                       CProps.SEVERITY: severity}
+
+        spec_list = mock_transformer.simple_collectd_alarm_generators(
+            update_vals=update_vals)
+        static_events = mock_transformer.generate_random_events_list(spec_list)
+
+        return static_events[0]
+
+    def _assert_collectd_vertex_equals(self,
+                                       collectd_vertex,
+                                       expected_time,
+                                       expected_resource_type,
+                                       expected_resource_name,
+                                       expected_severity):
+
+        self.assertEqual(format_unix_timestamp(expected_time),
+                         collectd_vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP])
+        self.assertEqual(expected_resource_type,
+                         collectd_vertex[VProps.VITRAGE_RESOURCE_TYPE])
+        self.assertEqual(expected_resource_name,
+                         collectd_vertex[CProps.RESOURCE_NAME])
+        self.assertEqual(expected_severity, collectd_vertex[VProps.SEVERITY])
+
+    def _assert_collectd_neighbor_equals(self,
+                                         collectd_neighbors,
+                                         expected_resource_type,
+                                         expected_resource_name):
+        self.assertEqual(1, len(collectd_neighbors))
+
+        self.assertEqual(expected_resource_type,
+                         collectd_neighbors[0][VProps.VITRAGE_TYPE])
+        self.assertEqual(expected_resource_name,
+                         collectd_neighbors[0][VProps.NAME])
+
+    def _assert_no_vertex(self, vertices):
+        self.assertTrue(len(vertices) == 0 or
+                        (len(vertices) == 1 and
+                         vertices[0].get(VProps.VITRAGE_IS_DELETED, False)))

vitrage/tests/unit/datasources/collectd/test_collectd_transformer.py

@@ -19,12 +19,14 @@ from oslo_config import cfg
 from vitrage.common.constants import DatasourceOpts as DSOpts
 from vitrage.common.constants import DatasourceProperties as DSProps
 from vitrage.common.constants import UpdateMethod
+from vitrage.common.constants import VertexProperties as VProps
 from vitrage.datasources.collectd import COLLECTD_DATASOURCE
 from vitrage.datasources.collectd.properties import \
     CollectdProperties as CProps
 from vitrage.datasources.collectd.transformer import CollectdTransformer
 from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
 from vitrage.datasources.nova.host.transformer import HostTransformer
+from vitrage.datasources.transformer_base import TransformerBase
 from vitrage.tests.mocks import mock_transformer
 from vitrage.tests.unit.datasources.test_alarm_transformer_base import \
     BaseAlarmTransformerTest
@@ -99,11 +101,17 @@ class TestCollectdTransformer(BaseAlarmTransformerTest):
 
     @staticmethod
     def _generate_event(time, hostname, severity):
+        # fake query result to be used by the transformer for determining
+        # the neighbor
+        query_result = [{VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE,
+                         VProps.ID: hostname}]
+
         update_vals = {CProps.HOST: hostname,
                        CProps.SEVERITY: severity,
                        CProps.TIME: time,
                        DSProps.SAMPLE_DATE: format_unix_timestamp(time),
-                       CProps.RESOURCE_NAME: hostname}
+                       CProps.RESOURCE_NAME: hostname,
+                       TransformerBase.QUERY_RESULT: query_result}
 
         generators = mock_transformer.simple_collectd_alarm_generators(
             update_vals=update_vals)