openstack/vitrage
Code Issues Proposed changes

Improve the consistency between code and document on value aggregation

Browse Source 
Change-Id: I6d215cae6a56af02294dfc32ae1b05fab8d77b15
This commit is contained in:
Yujun Zhang 2018-02-27 16:39:36 +00:00
parent 767049d064
commit 7b1a0b65b6
5 changed files with 136 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/source/contributor/alarm-severity-config.rst
View File

@ -45,13 +45,16 @@ The alarm severity configuration files configure how the severity of each
alarm is aggregated and normalized. There are several guidelines for creating
a config file:
- Severity comes with a priority, the higher severity "wins" in case
the same deduced alarm is raised in two scenarios (e.g., a host alarm caused
by both memory low and cpu high)
- Operational severity is a normalized severity values can be mapped to
specific defined values which can be found in operational_alarm_severity.py
(OperationalAlarmSeverity class).
- Aggregated severity comes with a priority, so that if an alarm is given
different severities from different sources (e.g., a host alarm raised both
by nagios and Vitrage), the severity with the highest priority will be used
as the **aggregated severity**.
- Aggregated severity is not used at the moment. It is designed for the use case
that an alarm is given different severities from different sources (e.g., a
host alarm raised both by nagios and Vitrage), the severity with the highest
priority will be used as the **aggregated severity**.
- The config file is in YAML format.
- The config filename must be <datasource name>.yaml, for the relevant
datasource.

102
vitrage/entity_graph/mappings/datasource_info_mapper.py
View File

@ -37,19 +37,19 @@ class DatasourceInfoMapper(object):
def __init__(self, conf):
self.conf = conf
self.category_normalizer = self._init_category_normalizer()
self.datasources_state_confs = self._load_state_configurations()
self.datasources_value_confs = self._load_value_configurations()
def vitrage_operational_state(self, datasource_name, state):
return self._get_state_data(datasource_name,
state,
def vitrage_operational_value(self, datasource_name, value):
return self._get_value_data(datasource_name,
value,
self.OPERATIONAL_VALUES)
def state_priority(self, datasource_name, state):
return self._get_state_data(datasource_name,
state,
def value_priority(self, datasource_name, value):
return self._get_value_data(datasource_name,
value,
self.PRIORITY_VALUES)
def vitrage_aggregated_state(self, new_vertex, graph_vertex):
def vitrage_aggregate_values(self, new_vertex, graph_vertex):
datasource_name = new_vertex[VProps.VITRAGE_TYPE] if \
VProps.VITRAGE_TYPE in new_vertex.properties else \
graph_vertex[VProps.VITRAGE_TYPE]
@ -58,32 +58,32 @@ class DatasourceInfoMapper(object):
VProps.VITRAGE_CATEGORY in new_vertex.properties else \
graph_vertex[VProps.VITRAGE_CATEGORY]
if datasource_name in self.datasources_state_confs or \
if datasource_name in self.datasources_value_confs or \
datasource_name not in self.conf.datasources.types:
value_properties = \
self.category_normalizer[vitrage_category].value_properties()
vitrage_operational_state, vitrage_aggregated_state, state_priority = \
self._find_operational_state_and_priority(new_vertex,
vitrage_operational_value, vitrage_aggregated_value, value_priority = \
self._find_operational_value_and_priority(new_vertex,
graph_vertex,
value_properties[0],
datasource_name)
value_properties.pop(0)
for property_ in value_properties:
t_operational_state, t_aggregated_state, t_state_priority = \
self._find_operational_state_and_priority(new_vertex,
t_operational_value, t_aggregated_value, t_value_priority = \
self._find_operational_value_and_priority(new_vertex,
graph_vertex,
property_,
datasource_name)
if t_state_priority > state_priority:
vitrage_operational_state = t_operational_state
vitrage_aggregated_state = t_aggregated_state
state_priority = t_state_priority
if t_value_priority > value_priority:
vitrage_operational_value = t_operational_value
vitrage_aggregated_value = t_aggregated_value
value_priority = t_value_priority
self.category_normalizer[vitrage_category].set_aggregated_value(
new_vertex, vitrage_aggregated_state)
new_vertex, vitrage_aggregated_value)
self.category_normalizer[vitrage_category].set_operational_value(
new_vertex, vitrage_operational_state)
new_vertex, vitrage_operational_value)
else:
self.category_normalizer[vitrage_category].set_aggregated_value(
new_vertex, self.UNDEFINED_DATASOURCE)
@ -92,12 +92,12 @@ class DatasourceInfoMapper(object):
def get_datasource_priorities(self, datasource_name=None):
if datasource_name:
datasource_info = self.datasources_state_confs[datasource_name]
datasource_info = self.datasources_value_confs[datasource_name]
return datasource_info[self.PRIORITY_VALUES]
else:
priorities_dict = \
{key: self.datasources_state_confs[key][self.PRIORITY_VALUES]
for key in self.datasources_state_confs.keys()}
{key: self.datasources_value_confs[key][self.PRIORITY_VALUES]
for key in self.datasources_value_confs.keys()}
return priorities_dict
@staticmethod
@ -107,7 +107,7 @@ class DatasourceInfoMapper(object):
EntityCategory.ALARM: AlarmHandler()
}
def _load_state_configurations(self):
def _load_value_configurations(self):
valid_datasources_conf = {}
erroneous_datasources_conf = []
@ -119,7 +119,7 @@ class DatasourceInfoMapper(object):
full_path = self.conf.entity_graph.datasources_values_dir \
+ '/' + file_name
operational_values, priority_values = \
self._retrieve_states_and_priorities_from_file(full_path)
self._retrieve_values_and_priorities_from_file(full_path)
valid_datasources_conf[os.path.splitext(file_name)[0]] = {
self.OPERATIONAL_VALUES: operational_values,
self.PRIORITY_VALUES: priority_values
@ -130,14 +130,14 @@ class DatasourceInfoMapper(object):
LOG.error('erroneous data sources is %s',
erroneous_datasources_conf.append(datasource))
self._check_state_confs_exists(
self._check_value_confs_exists(
[key for key in valid_datasources_conf.keys()],
erroneous_datasources_conf)
return valid_datasources_conf
def _retrieve_states_and_priorities_from_file(self, full_path):
states = {}
def _retrieve_values_and_priorities_from_file(self, full_path):
values = {}
priorities = {}
config = file_utils.load_yaml_file(full_path, with_exception=True)
vitrage_category = config['category']
@ -147,57 +147,57 @@ class DatasourceInfoMapper(object):
priority_value = int(aggregated_values['priority'])
# original to operational value
for original_state in aggregated_values['original values']:
original_value = original_state['name']
operational_value = original_state['operational_value']
states[original_value.upper()] = operational_value
priorities[original_value.upper()] = priority_value
for value_map in aggregated_values['original values']:
name = value_map['name']
operational_value = value_map['operational_value']
values[name.upper()] = operational_value
priorities[name.upper()] = priority_value
self._check_validity(vitrage_category, states, priorities, full_path)
self._check_validity(vitrage_category, values, priorities, full_path)
self._add_default_states(states, priorities, vitrage_category)
self._add_default_values(values, priorities, vitrage_category)
return states, priorities
return values, priorities
def _add_default_states(self, states, priorities, category):
def _add_default_values(self, values, priorities, category):
default_values = self.category_normalizer[category].default_values()
for original_val, operational_val, priority_val in default_values:
states[original_val] = operational_val
values[original_val] = operational_val
priorities[original_val] = priority_val
def _check_validity(self, category, states, priorities, full_path):
def _check_validity(self, category, values, priorities, full_path):
# check that all the operational values exists
state_class_instance = \
self.category_normalizer[category].get_value_class_instance()
operational_values = DatasourceInfoMapper.\
_get_all_local_variables_of_class(state_class_instance)
for operational_value in states.values():
for operational_value in values.values():
if operational_value not in operational_values:
raise ValueError('operational value %s for %s is not in %s',
operational_value, full_path,
state_class_instance.__class__.__name__)
def _get_state_data(self, datasource_name, state, data_type):
def _get_value_data(self, datasource_name, value, data_type):
try:
upper_state = state if not state else state.upper()
upper_value = value if not value else value.upper()
if datasource_name in self.datasources_state_confs:
values_conf = self.datasources_state_confs[
if datasource_name in self.datasources_value_confs:
values_conf = self.datasources_value_confs[
datasource_name][data_type]
return values_conf[upper_state] if upper_state in values_conf \
return values_conf[upper_value] if upper_value in values_conf \
else values_conf[None]
else:
values_conf = self.datasources_state_confs[
values_conf = self.datasources_value_confs[
DEFAULT_INFO_MAPPER][data_type]
return values_conf[upper_state] if upper_state in values_conf \
return values_conf[upper_value] if upper_value in values_conf \
else values_conf[None]
except Exception:
LOG.error('Exception in datasource: %s', datasource_name)
raise
def _find_operational_state_and_priority(self,
def _find_operational_value_and_priority(self,
new_vertex,
graph_vertex,
property_,
@ -208,20 +208,20 @@ class DatasourceInfoMapper(object):
upper_state = state if not state else state.upper()
vitrage_operational_state = self.vitrage_operational_state(
vitrage_operational_state = self.vitrage_operational_value(
datasource_name, upper_state)
state_priority = self.state_priority(datasource_name,
value_priority = self.value_priority(datasource_name,
upper_state)
return vitrage_operational_state, upper_state, state_priority
return vitrage_operational_state, upper_state, value_priority
@staticmethod
def _get_all_local_variables_of_class(class_instance):
return [getattr(class_instance, attr) for attr in dir(class_instance)
if not callable(attr) and not attr.startswith("__")]
def _check_state_confs_exists(self,
def _check_value_confs_exists(self,
ok_datasources,
error_datasources):

10
vitrage/entity_graph/processor/processor.py
View File

@ -36,7 +36,7 @@ class Processor(processor.ProcessorBase):
super(Processor, self).__init__()
self.conf = conf
self.transformer_manager = TransformerManager(self.conf)
self.state_manager = DatasourceInfoMapper(self.conf)
self.info_mapper = DatasourceInfoMapper(self.conf)
self._initialize_events_actions()
self.initialization_status = initialization_status
self.entity_graph = e_graph
@ -58,7 +58,7 @@ class Processor(processor.ProcessorBase):
self._enrich_event(event)
entity = self.transformer_manager.transform(event)
self._calculate_vitrage_aggregated_state(entity.vertex, entity.action)
self._calculate_vitrage_aggregated_values(entity.vertex, entity.action)
self.actions[entity.action](entity.vertex, entity.neighbors)
if self._graph_persistor:
self._graph_persistor.update_last_event_timestamp(event)
@ -223,7 +223,7 @@ class Processor(processor.ProcessorBase):
if not graph_vertex or not PUtils.is_deleted(graph_vertex):
if PUtils.can_update_vertex(graph_vertex, vertex):
LOG.debug("Updates vertex: %s", vertex)
self._calculate_vitrage_aggregated_state(vertex, action)
self._calculate_vitrage_aggregated_values(vertex, action)
PUtils.update_entity_graph_vertex(self.entity_graph,
graph_vertex,
vertex)
@ -301,7 +301,7 @@ class Processor(processor.ProcessorBase):
GraphAction.END_MESSAGE: self.handle_end_message
}
def _calculate_vitrage_aggregated_state(self, vertex, action):
def _calculate_vitrage_aggregated_values(self, vertex, action):
LOG.debug("calculate event state")
try:
@ -320,7 +320,7 @@ class Processor(processor.ProcessorBase):
action, vertex)
return None
self.state_manager.vitrage_aggregated_state(vertex, graph_vertex)
self.info_mapper.vitrage_aggregate_values(vertex, graph_vertex)
except Exception as e:
LOG.exception("Calculate aggregated state failed - %s", e)

12
vitrage/tests/unit/entity_graph/processor/test_processor.py
View File

@ -322,7 +322,7 @@ class TestProcessor(TestEntityGraphUnitBase):
# action
# state already exists and its updated
instances[0][0][VProps.STATE] = 'SUSPENDED'
instances[0][1]._calculate_vitrage_aggregated_state(
instances[0][1]._calculate_vitrage_aggregated_values(
instances[0][0], GraphAction.UPDATE_ENTITY)
# vitrage state doesn't exist and its updated
@ -330,13 +330,13 @@ class TestProcessor(TestEntityGraphUnitBase):
instances[1][1].entity_graph.update_vertex(instances[1][0])
instances[1][0][VProps.VITRAGE_STATE] = \
OperationalResourceState.SUBOPTIMAL
instances[1][1]._calculate_vitrage_aggregated_state(
instances[1][1]._calculate_vitrage_aggregated_values(
instances[1][0], GraphAction.UPDATE_ENTITY)
# state exists and vitrage state changes
instances[2][0][VProps.VITRAGE_STATE] = \
OperationalResourceState.SUBOPTIMAL
instances[2][1]._calculate_vitrage_aggregated_state(
instances[2][1]._calculate_vitrage_aggregated_values(
instances[2][0], GraphAction.UPDATE_ENTITY)
# vitrage state exists and state changes
@ -345,7 +345,7 @@ class TestProcessor(TestEntityGraphUnitBase):
OperationalResourceState.SUBOPTIMAL
instances[3][1].entity_graph.update_vertex(instances[3][0])
instances[3][0][VProps.STATE] = 'SUSPENDED'
instances[3][1]._calculate_vitrage_aggregated_state(
instances[3][1]._calculate_vitrage_aggregated_values(
instances[3][0], GraphAction.UPDATE_ENTITY)
# state and vitrage state exists and state changes
@ -353,14 +353,14 @@ class TestProcessor(TestEntityGraphUnitBase):
OperationalResourceState.SUBOPTIMAL
instances[4][1].entity_graph.update_vertex(instances[4][0])
instances[4][0][VProps.STATE] = 'SUSPENDED'
instances[4][1]._calculate_vitrage_aggregated_state(
instances[4][1]._calculate_vitrage_aggregated_values(
instances[4][0], GraphAction.UPDATE_ENTITY)
# state and vitrage state exists and vitrage state changes
instances[5][0][VProps.VITRAGE_STATE] = \
OperationalResourceState.SUBOPTIMAL
instances[5][1].entity_graph.update_vertex(instances[5][0])
instances[5][1]._calculate_vitrage_aggregated_state(
instances[5][1]._calculate_vitrage_aggregated_values(
instances[5][0], GraphAction.UPDATE_ENTITY)
# test assertions

135
vitrage/tests/unit/entity_graph/states/test_datasource_info_mapper.py
View File

@ -67,18 +67,18 @@ class TestDatasourceInfoMapper(base.BaseTest):
cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
cls._load_datasources(cls.conf)
def test_load_datasource_state_without_errors(self):
def test_load_datasource_value_without_errors(self):
# action
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# test assertions
# Total datasources plus the evaluator which is not definable
total_datasources = len(self.conf.datasources.types) + 1
self.assertThat(state_manager.datasources_state_confs,
self.assertThat(info_mapper.datasources_value_confs,
matchers.HasLength(total_datasources))
def test_load_datasources_state_with_errors(self):
def test_load_datasources_value_with_errors(self):
# setup
entity_graph_opts = [
cfg.StrOpt('datasources_values_dir',
@ -91,108 +91,107 @@ class TestDatasourceInfoMapper(base.BaseTest):
self._load_datasources(conf)
# action
state_manager = DatasourceInfoMapper(conf)
info_mapper = DatasourceInfoMapper(conf)
# test assertions
missing_states = 1
missing_values = 1
erroneous_values = 1
num_valid_datasources = len(state_manager.datasources_state_confs) + \
missing_states + erroneous_values
num_valid_datasources = len(info_mapper.datasources_value_confs) + \
missing_values + erroneous_values
self.assertThat(conf.datasources.types,
matchers.HasLength(num_valid_datasources))
def test_vitrage_operational_state_exists(self):
def test_vitrage_operational_value_exists(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# action
vitrage_operational_state = \
state_manager.vitrage_operational_state(NOVA_INSTANCE_DATASOURCE,
'BUILDING')
vitrage_operational_value = \
info_mapper.vitrage_operational_value(NOVA_INSTANCE_DATASOURCE,
'BUILDING')
# test assertions
self.assertEqual(OperationalResourceState.TRANSIENT,
vitrage_operational_state)
vitrage_operational_value)
def test_vitrage_operational_state_not_exists(self):
def test_vitrage_operational_value_not_exists(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# action
vitrage_operational_state = \
state_manager.vitrage_operational_state(NOVA_INSTANCE_DATASOURCE,
'NON EXISTING STATE')
vitrage_operational_value = \
info_mapper.vitrage_operational_value(NOVA_INSTANCE_DATASOURCE,
'NON EXISTING STATE')
# test assertions
self.assertEqual(OperationalResourceState.NA,
vitrage_operational_state)
vitrage_operational_value)
def test_vitrage_operational_state_DS_not_exists_and_state_not_exist(self):
def test_vitrage_operational_value_DS_not_exists_and_value_not_exist(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# action
vitrage_operational_state = \
state_manager.vitrage_operational_state('NON EXISTING DATASOURCE',
'BUILDING')
vitrage_operational_value = \
info_mapper.vitrage_operational_value('NON EXISTING DATASOURCE',
'BUILDING')
# test assertions
self.assertEqual(OperationalResourceState.NA,
vitrage_operational_state)
vitrage_operational_value)
def test_vitrage_operational_state_DS_not_exists_and_state_exist(self):
def test_vitrage_operational_value_DS_not_exists_and_value_exist(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# action
vitrage_operational_state = \
state_manager.vitrage_operational_state('NON EXISTING DATASOURCE',
'AVAILABLE')
vitrage_operational_value = \
info_mapper.vitrage_operational_value('NON EXISTING DATASOURCE',
'AVAILABLE')
# test assertions
self.assertEqual(OperationalResourceState.OK,
vitrage_operational_state)
vitrage_operational_value)
def test_state_priority(self):
def test_value_priority(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# action
state_priority = \
state_manager.state_priority(NOVA_INSTANCE_DATASOURCE,
'ACTIVE')
value_priority = \
info_mapper.value_priority(NOVA_INSTANCE_DATASOURCE, 'ACTIVE')
# test assertions
self.assertEqual(10, state_priority)
self.assertEqual(10, value_priority)
def test_state_priority_not_exists(self):
def test_value_priority_not_exists(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# action
state_priority = \
state_manager.state_priority(NOVA_INSTANCE_DATASOURCE,
'NON EXISTING STATE')
value_priority = \
info_mapper.value_priority(NOVA_INSTANCE_DATASOURCE,
'NON EXISTING STATE')
# test assertions
self.assertEqual(0, state_priority)
self.assertEqual(0, value_priority)
def test_state_priority_datasource_not_exists(self):
def test_value_priority_datasource_not_exists(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
# action
state_priority = \
state_manager.state_priority('NON EXISTING DATASOURCE',
'ACTIVE')
value_priority = \
info_mapper.value_priority('NON EXISTING DATASOURCE',
'ACTIVE')
# test assertions
self.assertEqual(10,
state_priority)
value_priority)
def test_vitrage_aggregated_state(self):
def test_vitrage_aggregated_value(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
metadata1 = {VProps.VITRAGE_STATE: 'SUSPENDED'}
new_vertex1 = create_vertex('12345',
vitrage_category=EntityCategory.RESOURCE,
@ -207,8 +206,8 @@ class TestDatasourceInfoMapper(base.BaseTest):
metadata=metadata2)
# action
state_manager.vitrage_aggregated_state(new_vertex1, None)
state_manager.vitrage_aggregated_state(new_vertex2, None)
info_mapper.vitrage_aggregate_values(new_vertex1, None)
info_mapper.vitrage_aggregate_values(new_vertex2, None)
# test assertions
self.assertEqual('SUSPENDED',
@ -220,9 +219,9 @@ class TestDatasourceInfoMapper(base.BaseTest):
self.assertEqual(OperationalResourceState.SUBOPTIMAL,
new_vertex2[VProps.VITRAGE_OPERATIONAL_STATE])
def test_vitrage_aggregated_state_functionalities(self):
def test_vitrage_aggregated_value_functionalities(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
new_vertex1 = create_vertex('12345',
vitrage_category=EntityCategory.RESOURCE,
vitrage_type=NOVA_INSTANCE_DATASOURCE,
@ -244,12 +243,12 @@ class TestDatasourceInfoMapper(base.BaseTest):
metadata=metadata3)
# action
state_manager.vitrage_aggregated_state(new_vertex1,
None)
state_manager.vitrage_aggregated_state(new_vertex2,
None)
state_manager.vitrage_aggregated_state(new_vertex3,
graph_vertex3)
info_mapper.vitrage_aggregate_values(new_vertex1,
None)
info_mapper.vitrage_aggregate_values(new_vertex2,
None)
info_mapper.vitrage_aggregate_values(new_vertex3,
graph_vertex3)
# test assertions
self.assertEqual('ACTIVE',
@ -265,9 +264,9 @@ class TestDatasourceInfoMapper(base.BaseTest):
self.assertEqual(OperationalResourceState.SUBOPTIMAL,
new_vertex3[VProps.VITRAGE_OPERATIONAL_STATE])
def test_vitrage_aggregated_state_datasource_not_exists(self):
def test_vitrage_aggregated_value_datasource_not_exists(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
metadata = {VProps.VITRAGE_STATE: 'SUSPENDED'}
new_vertex = create_vertex('12345',
vitrage_category=EntityCategory.RESOURCE,
@ -276,7 +275,7 @@ class TestDatasourceInfoMapper(base.BaseTest):
metadata=metadata)
# action
state_manager.vitrage_aggregated_state(new_vertex, None)
info_mapper.vitrage_aggregate_values(new_vertex, None)
# test assertions
self.assertEqual('ACTIVE',
@ -284,9 +283,9 @@ class TestDatasourceInfoMapper(base.BaseTest):
self.assertEqual(OperationalResourceState.OK,
new_vertex[VProps.VITRAGE_OPERATIONAL_STATE])
def test_vitrage_aggregated_state_DS_not_exists_and_wrong_state(self):
def test_vitrage_aggregated_value_DS_not_exists_and_wrong_state(self):
# setup
state_manager = DatasourceInfoMapper(self.conf)
info_mapper = DatasourceInfoMapper(self.conf)
metadata = {VProps.VITRAGE_STATE: 'SUSPENDED'}
new_vertex = create_vertex('12345',
vitrage_category=EntityCategory.RESOURCE,
@ -295,7 +294,7 @@ class TestDatasourceInfoMapper(base.BaseTest):
metadata=metadata)
# action
state_manager.vitrage_aggregated_state(new_vertex, None)
info_mapper.vitrage_aggregate_values(new_vertex, None)
# test assertions
self.assertEqual('NON EXISTING STATE',