Support different resource types in collectd

Currently, due to a bug, collectd alarms are supported only on hosts. I fixed the bug + added tests

Change-Id: Ie2760689e5cd248f6d499f2815e86d36c011dc6e
This commit is contained in:
Ifat Afek 2017-07-10 16:49:46 +00:00
parent e0da4a0768
commit 1c49907ada
5 changed files with 246 additions and 14 deletions

View File

@ -38,6 +38,7 @@ class VertexProperties(object):
GRAPH_INDEX = 'graph_index' GRAPH_INDEX = 'graph_index'
RAWTEXT = 'rawtext' RAWTEXT = 'rawtext'
RESOURCE_ID = 'resource_id' RESOURCE_ID = 'resource_id'
RESOURCE_NAME = 'resource_name'
VITRAGE_RESOURCE_ID = 'vitrage_resource_id' VITRAGE_RESOURCE_ID = 'vitrage_resource_id'
VITRAGE_RESOURCE_TYPE = 'vitrage_resource_type' VITRAGE_RESOURCE_TYPE = 'vitrage_resource_type'
RESOURCE = 'resource' RESOURCE = 'resource'

View File

@ -14,7 +14,7 @@
from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EdgeLabel from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory from vitrage.common.constants import EntityCategory as Category
from vitrage.common.constants import VertexProperties as VProps from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.alarm_properties import AlarmProperties as AlarmProps from vitrage.datasources.alarm_properties import AlarmProperties as AlarmProps
from vitrage.datasources.alarm_transformer_base import AlarmTransformerBase from vitrage.datasources.alarm_transformer_base import AlarmTransformerBase
@ -54,12 +54,12 @@ class CollectdTransformer(AlarmTransformerBase):
VProps.NAME: entity_event[CProps.MESSAGE], VProps.NAME: entity_event[CProps.MESSAGE],
VProps.SEVERITY: entity_event[CProps.SEVERITY], VProps.SEVERITY: entity_event[CProps.SEVERITY],
VProps.RAWTEXT: self.generate_raw_text(entity_event), VProps.RAWTEXT: self.generate_raw_text(entity_event),
VProps.RESOURCE_ID: entity_event[CProps.RESOURCE_NAME] VProps.RESOURCE_NAME: entity_event[CProps.RESOURCE_NAME]
} }
return graph_utils.create_vertex( return graph_utils.create_vertex(
self._create_entity_key(entity_event), self._create_entity_key(entity_event),
vitrage_category=EntityCategory.ALARM, vitrage_category=Category.ALARM,
vitrage_type=entity_event[DSProps.ENTITY_TYPE], vitrage_type=entity_event[DSProps.ENTITY_TYPE],
vitrage_sample_timestamp=vitrage_sample_timestamp, vitrage_sample_timestamp=vitrage_sample_timestamp,
entity_state=entity_state, entity_state=entity_state,
@ -73,17 +73,14 @@ class CollectdTransformer(AlarmTransformerBase):
return self._create_collectd_neighbors(entity_event) return self._create_collectd_neighbors(entity_event)
def _create_collectd_neighbors(self, entity_event): def _create_collectd_neighbors(self, entity_event):
graph_neighbors = entity_event.get(self.QUERY_RESULT, [])
resource_type = entity_event[CProps.RESOURCE_TYPE] return [self._create_neighbor(entity_event,
if resource_type: graph_neighbor[VProps.ID],
return [self._create_neighbor( graph_neighbor[VProps.VITRAGE_TYPE],
entity_event, EdgeLabel.ON,
entity_event[CProps.RESOURCE_NAME], neighbor_category=Category.RESOURCE)
resource_type, for graph_neighbor in graph_neighbors]
EdgeLabel.ON,
neighbor_category=EntityCategory.RESOURCE)]
return []
def _ok_status(self, entity_event): def _ok_status(self, entity_event):
return entity_event[CProps.SEVERITY] == 'OK' return entity_event[CProps.SEVERITY] == 'OK'
@ -106,3 +103,14 @@ class CollectdTransformer(AlarmTransformerBase):
entity_event[CProps.PLUGIN], entity_event[CProps.PLUGIN],
entity_event.get(CProps.PLUGIN_INSTANCE)] entity_event.get(CProps.PLUGIN_INSTANCE)]
return '-'.join([resource for resource in resources if resource]) return '-'.join([resource for resource in resources if resource])
@staticmethod
def get_enrich_query(event):
resource_type = event.get(CProps.RESOURCE_TYPE)
resource_name = event.get(CProps.RESOURCE_NAME)
if resource_type and resource_name:
return {VProps.NAME: resource_name,
VProps.VITRAGE_TYPE: resource_type}
return None

View File

@ -0,0 +1,15 @@
# Copyright 2017 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
__author__ = 'stack'

View File

@ -0,0 +1,200 @@
# Copyright 2017 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import time
from oslo_config import cfg
from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.collectd import COLLECTD_DATASOURCE
from vitrage.datasources.collectd.properties import \
CollectdProperties as CProps
from vitrage.datasources import NOVA_HOST_DATASOURCE
from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
from vitrage.datasources import NOVA_ZONE_DATASOURCE
from vitrage.tests.functional.datasources.base import \
TestDataSourcesBase
from vitrage.tests.mocks import mock_transformer
from vitrage.utils.datetime import format_unix_timestamp
class TestCollectd(TestDataSourcesBase):
DATASOURCES_OPTS = [
cfg.ListOpt('types',
default=[COLLECTD_DATASOURCE,
NOVA_HOST_DATASOURCE,
NOVA_INSTANCE_DATASOURCE,
NOVA_ZONE_DATASOURCE],
help='Names of supported driver data sources'),
cfg.ListOpt('path',
default=['vitrage.datasources'],
help='base path for data sources')
]
# noinspection PyPep8Naming
@classmethod
def setUpClass(cls):
super(TestCollectd, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
cls.load_datasources(cls.conf)
def test_collectd_alarm_on_host(self):
self._test_collectd_alarm(NOVA_HOST_DATASOURCE, 'host-2', 'host-2')
def test_collectd_alarm_on_instance(self):
self._test_collectd_alarm(NOVA_INSTANCE_DATASOURCE, 'vm-5', 'host-4')
def _test_collectd_alarm(self, resource_type, resource_name, host_name):
# Setup
processor = self._create_processor_with_graph(self.conf, uuid=True)
self.assertEqual(self._num_total_expected_vertices(),
len(processor.entity_graph))
time1 = time.time()
severity1 = 'WARNING'
collectd_event = self._create_collectd_event(time1,
resource_type,
resource_name,
host_name,
severity1)
# Action
processor.process_event(collectd_event)
# Test assertions
self.assertEqual(self._num_total_expected_vertices() + 1,
len(processor.entity_graph))
collectd_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter={
VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE
})
self.assertEqual(1, len(collectd_vertices))
collectd_vertex1 = collectd_vertices[0]
self._assert_collectd_vertex_equals(collectd_vertex1,
time1,
resource_type,
resource_name,
severity1)
collectd_neighbors = processor.entity_graph.neighbors(
collectd_vertices[0].vertex_id)
self._assert_collectd_neighbor_equals(collectd_neighbors,
resource_type,
resource_name)
# Action 2 - update the existing alarm
time2 = time.time()
severity2 = 'ERROR'
collectd_event = self._create_collectd_event(time2,
resource_type,
resource_name,
host_name,
severity2)
processor.process_event(collectd_event)
# Test assertions - the collectd alarm vertex should be the same
self.assertEqual(self._num_total_expected_vertices() + 1,
len(processor.entity_graph))
collectd_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter={
VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE
})
self.assertEqual(1, len(collectd_vertices))
collectd_vertex2 = collectd_vertices[0]
self.assertEqual(collectd_vertex1[VProps.VITRAGE_ID],
collectd_vertex2[VProps.VITRAGE_ID])
# Action 3 - clear the alarm
time3 = time.time()
severity3 = 'OK'
collectd_event = self._create_collectd_event(time3,
resource_type,
resource_name,
host_name,
severity3)
processor.process_event(collectd_event)
# Test assertions - the collectd alarm vertex should be removed
collectd_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter={
VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
VProps.VITRAGE_TYPE: COLLECTD_DATASOURCE
})
self._assert_no_vertex(collectd_vertices)
@staticmethod
def _create_collectd_event(time,
resource_type,
resource_name,
host_name,
severity):
update_vals = {CProps.TIME: time,
DSProps.SAMPLE_DATE: format_unix_timestamp(time),
CProps.HOST: host_name,
CProps.RESOURCE_TYPE: resource_type,
CProps.RESOURCE_NAME: resource_name,
CProps.MESSAGE: 'A message for you',
CProps.SEVERITY: severity}
spec_list = mock_transformer.simple_collectd_alarm_generators(
update_vals=update_vals)
static_events = mock_transformer.generate_random_events_list(spec_list)
return static_events[0]
def _assert_collectd_vertex_equals(self,
collectd_vertex,
expected_time,
expected_resource_type,
expected_resource_name,
expected_severity):
self.assertEqual(format_unix_timestamp(expected_time),
collectd_vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP])
self.assertEqual(expected_resource_type,
collectd_vertex[VProps.VITRAGE_RESOURCE_TYPE])
self.assertEqual(expected_resource_name,
collectd_vertex[CProps.RESOURCE_NAME])
self.assertEqual(expected_severity, collectd_vertex[VProps.SEVERITY])
def _assert_collectd_neighbor_equals(self,
collectd_neighbors,
expected_resource_type,
expected_resource_name):
self.assertEqual(1, len(collectd_neighbors))
self.assertEqual(expected_resource_type,
collectd_neighbors[0][VProps.VITRAGE_TYPE])
self.assertEqual(expected_resource_name,
collectd_neighbors[0][VProps.NAME])
def _assert_no_vertex(self, vertices):
self.assertTrue(len(vertices) == 0 or
(len(vertices) == 1 and
vertices[0].get(VProps.VITRAGE_IS_DELETED, False)))

View File

@ -19,12 +19,14 @@ from oslo_config import cfg
from vitrage.common.constants import DatasourceOpts as DSOpts from vitrage.common.constants import DatasourceOpts as DSOpts
from vitrage.common.constants import DatasourceProperties as DSProps from vitrage.common.constants import DatasourceProperties as DSProps
from vitrage.common.constants import UpdateMethod from vitrage.common.constants import UpdateMethod
from vitrage.common.constants import VertexProperties as VProps
from vitrage.datasources.collectd import COLLECTD_DATASOURCE from vitrage.datasources.collectd import COLLECTD_DATASOURCE
from vitrage.datasources.collectd.properties import \ from vitrage.datasources.collectd.properties import \
CollectdProperties as CProps CollectdProperties as CProps
from vitrage.datasources.collectd.transformer import CollectdTransformer from vitrage.datasources.collectd.transformer import CollectdTransformer
from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE from vitrage.datasources.nova.host import NOVA_HOST_DATASOURCE
from vitrage.datasources.nova.host.transformer import HostTransformer from vitrage.datasources.nova.host.transformer import HostTransformer
from vitrage.datasources.transformer_base import TransformerBase
from vitrage.tests.mocks import mock_transformer from vitrage.tests.mocks import mock_transformer
from vitrage.tests.unit.datasources.test_alarm_transformer_base import \ from vitrage.tests.unit.datasources.test_alarm_transformer_base import \
BaseAlarmTransformerTest BaseAlarmTransformerTest
@ -99,11 +101,17 @@ class TestCollectdTransformer(BaseAlarmTransformerTest):
@staticmethod @staticmethod
def _generate_event(time, hostname, severity): def _generate_event(time, hostname, severity):
# fake query result to be used by the transformer for determining
# the neighbor
query_result = [{VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE,
VProps.ID: hostname}]
update_vals = {CProps.HOST: hostname, update_vals = {CProps.HOST: hostname,
CProps.SEVERITY: severity, CProps.SEVERITY: severity,
CProps.TIME: time, CProps.TIME: time,
DSProps.SAMPLE_DATE: format_unix_timestamp(time), DSProps.SAMPLE_DATE: format_unix_timestamp(time),
CProps.RESOURCE_NAME: hostname} CProps.RESOURCE_NAME: hostname,
TransformerBase.QUERY_RESULT: query_result}
generators = mock_transformer.simple_collectd_alarm_generators( generators = mock_transformer.simple_collectd_alarm_generators(
update_vals=update_vals) update_vals=update_vals)