From fec7f59687b85d13d96afa823671bb7faa77d4e6 Mon Sep 17 00:00:00 2001 From: Ifat Afek Date: Thu, 7 Dec 2017 09:08:29 +0000 Subject: [PATCH] Refactor template loading The template loading code was extracted from the TemplateData class and placed in separate files, to allow future support of per-version loading (i.e. different loaders for different template versions). No real change was done in the code, it was just split to separate files. The changes: * Separate the loading logic from the data * Split the template loading to specific loaders: template, scenario and equivalences Change-Id: I810da2682f3c1804312b26041b73280c040432a5 Implements: blueprint refactor-execute-mistral-definition --- vitrage/evaluator/condition.py | 1 - vitrage/evaluator/equivalence_repository.py | 7 +- vitrage/evaluator/scenario_repository.py | 14 +- vitrage/evaluator/template_data.py | 385 +----------------- .../evaluator/template_loading/__init__.py | 14 + .../equivalence_loader.py} | 6 +- .../template_loading/props_converter.py | 49 +++ .../template_loading/scenario_loader.py | 165 ++++++++ .../template_loading/subgraph_builder.py | 76 ++++ .../template_loading/template_loader.py | 164 ++++++++ .../tests/unit/evaluator/test_condition.py | 6 +- ...nce_data.py => test_equivalence_loader.py} | 6 +- ...mplate_data.py => test_template_loader.py} | 15 +- 13 files changed, 506 insertions(+), 402 deletions(-) create mode 100644 vitrage/evaluator/template_loading/__init__.py rename vitrage/evaluator/{equivalence_data.py => template_loading/equivalence_loader.py} (91%) create mode 100644 vitrage/evaluator/template_loading/props_converter.py create mode 100644 vitrage/evaluator/template_loading/scenario_loader.py create mode 100644 vitrage/evaluator/template_loading/subgraph_builder.py create mode 100644 vitrage/evaluator/template_loading/template_loader.py rename vitrage/tests/unit/evaluator/{test_equivalence_data.py => test_equivalence_loader.py} (92%) rename vitrage/tests/unit/evaluator/{test_template_data.py => test_template_loader.py} (96%) diff --git a/vitrage/evaluator/condition.py b/vitrage/evaluator/condition.py index 10ce0d654..9d7c60a15 100644 --- a/vitrage/evaluator/condition.py +++ b/vitrage/evaluator/condition.py @@ -22,7 +22,6 @@ from sympy import Symbol ConditionVar = namedtuple('ConditionVar', ['symbol_name', 'positive']) -EdgeDescription = namedtuple('EdgeDescription', ['edge', 'source', 'target']) class SymbolResolver(object): diff --git a/vitrage/evaluator/equivalence_repository.py b/vitrage/evaluator/equivalence_repository.py index dc1076423..36c684468 100644 --- a/vitrage/evaluator/equivalence_repository.py +++ b/vitrage/evaluator/equivalence_repository.py @@ -13,7 +13,8 @@ # under the License. from vitrage.common.exception import VitrageError -from vitrage.evaluator.equivalence_data import EquivalenceData +from vitrage.evaluator.template_loading.equivalence_loader import \ + EquivalenceLoader from vitrage.utils import file as file_utils @@ -25,8 +26,8 @@ class EquivalenceRepository(object): equivalence_defs = file_utils.load_yaml_files(directory) for equivalence_def in equivalence_defs: - equivalence_data = EquivalenceData(equivalence_def) - for equivalence in equivalence_data.equivalences: + equivalences = EquivalenceLoader(equivalence_def).equivalences + for equivalence in equivalences: self._add_equivalence(equivalence) return self.entity_equivalences diff --git a/vitrage/evaluator/scenario_repository.py b/vitrage/evaluator/scenario_repository.py index c5b76a47a..efd793e39 100644 --- a/vitrage/evaluator/scenario_repository.py +++ b/vitrage/evaluator/scenario_repository.py @@ -22,8 +22,9 @@ from oslo_utils import uuidutils from vitrage.common.utils import get_portion from vitrage.evaluator.base import Template from vitrage.evaluator.equivalence_repository import EquivalenceRepository -from vitrage.evaluator.template_data import TemplateData from vitrage.evaluator.template_fields import TemplateFields +from vitrage.evaluator.template_loading.scenario_loader import ScenarioLoader +from vitrage.evaluator.template_loading.template_loader import TemplateLoader from vitrage.evaluator.template_validation.content.definitions_validator \ import DefinitionsValidator as DefValidator from vitrage.evaluator.template_validation.content.template_content_validator \ @@ -127,7 +128,8 @@ class ScenarioRepository(object): current_time, result) if result.is_valid_config: - template_data = TemplateData(template_def, self._def_templates) + template_data = \ + TemplateLoader().load(template_def, self._def_templates) for scenario in template_data.scenarios: for equivalent_scenario in self._expand_equivalence(scenario): self._add_scenario(equivalent_scenario) @@ -168,10 +170,10 @@ class ScenarioRepository(object): scenarios_out = list(scenarios_in) for entity_key in entity_keys: for scenario in scenarios_in: - equivalent_scenario = TemplateData.ScenarioData. \ - build_equivalent_scenario(scenario, - symbol_name, - entity_key) + equivalent_scenario = \ + ScenarioLoader.build_equivalent_scenario(scenario, + symbol_name, + entity_key) scenarios_out.append(equivalent_scenario) return scenarios_out diff --git a/vitrage/evaluator/template_data.py b/vitrage/evaluator/template_data.py index 97184f9ff..50981515e 100644 --- a/vitrage/evaluator/template_data.py +++ b/vitrage/evaluator/template_data.py @@ -15,22 +15,12 @@ from collections import namedtuple -from vitrage.common.constants import EdgeProperties as EProps -from vitrage.common.constants import VertexProperties as VProps -from vitrage.common.exception import VitrageError -from vitrage.evaluator.condition import EdgeDescription -from vitrage.evaluator.condition import get_condition_common_targets -from vitrage.evaluator.condition import parse_condition -from vitrage.evaluator.condition import SymbolResolver -from vitrage.evaluator.template_fields import TemplateFields as TFields -from vitrage.graph.algo_driver.sub_graph_matching import NEG_CONDITION -from vitrage.graph.driver.networkx_graph import NXGraph -from vitrage.graph import Edge -from vitrage.graph import Vertex -from vitrage.utils import evaluator as evaluator_utils - ActionSpecs = namedtuple( 'ActionSpecs', ['id', 'type', 'targets', 'properties']) +EdgeDescription = namedtuple('EdgeDescription', ['edge', 'source', 'target']) + +ENTITY = 'entity' +RELATIONSHIP = 'relationship' class Scenario(object): @@ -52,63 +42,15 @@ class Scenario(object): self.entities == other.entities and \ self.relationships == other.relationships -ENTITY = 'entity' -RELATIONSHIP = 'relationship' - - -def copy_edge_desc(edge_desc): - return EdgeDescription(edge=edge_desc.edge.copy(), - source=edge_desc.source.copy(), - target=edge_desc.target.copy()) - # noinspection PyAttributeOutsideInit class TemplateData(object): - PROPS_CONVERSION = { - 'category': VProps.VITRAGE_CATEGORY, - 'type': VProps.VITRAGE_TYPE, - 'resource_id': VProps.VITRAGE_RESOURCE_ID, - 'sample_timestamp': VProps.VITRAGE_SAMPLE_TIMESTAMP, - 'is_deleted': VProps.VITRAGE_IS_DELETED, - 'is_placeholder': VProps.VITRAGE_IS_PLACEHOLDER, - 'aggregated_state': VProps.VITRAGE_AGGREGATED_STATE, - 'operational_state': VProps.VITRAGE_OPERATIONAL_STATE, - 'aggregated_severity': VProps.VITRAGE_AGGREGATED_SEVERITY, - 'operational_severity': VProps.VITRAGE_OPERATIONAL_SEVERITY - } - - def __init__(self, template_def, def_templates=None): - - if def_templates is None: - def_templates = {} - - self.name = template_def[TFields.METADATA][TFields.NAME] - defs = {} - self.entities = {} - if TFields.DEFINITIONS in template_def: - defs = template_def[TFields.DEFINITIONS] - if TFields.ENTITIES in defs: - self.entities = self._build_entities(defs[TFields.ENTITIES]) - self.relationships = {} - - # Add definitions from template then from definition templates. - if TFields.INCLUDES in template_def: - includes = template_def[TFields.INCLUDES] - self._build_entities_from_def_templates( - includes, def_templates, self.entities) - - if TFields.RELATIONSHIPS in defs: - self.relationships = self._build_relationships( - defs[TFields.RELATIONSHIPS]) - - if TFields.INCLUDES in template_def: - includes = template_def[TFields.INCLUDES] - self._build_relationships_with_def_templates(includes, - def_templates, - self.relationships) - - self.scenarios = self._build_scenarios(template_def[TFields.SCENARIOS]) + def __init__(self, name, entities, relationships, scenarios): + self.name = name + self.entities = entities + self.relationships = relationships + self.scenarios = scenarios @property def name(self): @@ -141,312 +83,3 @@ class TemplateData(object): @scenarios.setter def scenarios(self, scenarios): self._scenarios = scenarios - - def _build_entities(self, entities_defs): - - entities = {} - for entity_def in entities_defs: - - entity_dict = entity_def[TFields.ENTITY] - template_id = entity_dict[TFields.TEMPLATE_ID] - properties = self._convert_properties_with_dictionary( - self._extract_properties(entity_dict)) - entities[template_id] = Vertex(template_id, properties) - - return entities - - def _build_entities_from_def_templates( - self, includes, def_templates, entities): - - for def_template_dict in includes: - - name = def_template_dict[TFields.NAME] - def_template = evaluator_utils.find_def_template( - name, def_templates) - defs = def_template[TFields.DEFINITIONS] - entities_defs = defs[TFields.ENTITIES] - - for entity_def in entities_defs: - - entity_dict = entity_def[TFields.ENTITY] - template_id = entity_dict[TFields.TEMPLATE_ID] - if template_id not in entities: - - properties = self._convert_properties_with_dictionary( - self._extract_properties(entity_dict)) - entities[template_id] = Vertex(template_id, properties) - - def _build_relationships(self, relationships_defs): - - relationships = {} - for relationship_def in relationships_defs: - - relationship_dict = relationship_def[TFields.RELATIONSHIP] - relationship = self._extract_relationship_info(relationship_dict) - template_id = relationship_dict[TFields.TEMPLATE_ID] - relationships[template_id] = relationship - - return relationships - - def _build_relationships_with_def_templates( - self, includes, def_templates, relationships): - - for def_template_dict in includes: - - name = def_template_dict[TFields.NAME] - def_template = evaluator_utils.find_def_template( - name, def_templates) - - if TFields.RELATIONSHIPS in def_template[TFields.DEFINITIONS]: - defs = def_template[TFields.DEFINITIONS] - relationship_defs = defs[TFields.RELATIONSHIPS] - - for relationship_def in relationship_defs: - relationship_dict = relationship_def[TFields.RELATIONSHIP] - template_id = relationship_dict[TFields.TEMPLATE_ID] - - if template_id not in relationships: - relationship = self._extract_relationship_info( - relationship_dict) - relationships[template_id] = relationship - - def _extract_relationship_info(self, relationship_dict): - - source_id = relationship_dict[TFields.SOURCE] - target_id = relationship_dict[TFields.TARGET] - - edge = Edge(source_id, - target_id, - relationship_dict[TFields.RELATIONSHIP_TYPE], - self._extract_properties(relationship_dict)) - - source = self.entities[source_id] - target = self.entities[target_id] - return EdgeDescription(edge, source, target) - - @staticmethod - def _extract_properties(var_dict): - - ignore_ids = [TFields.TEMPLATE_ID, TFields.SOURCE, TFields.TARGET] - return \ - {key: var_dict[key] for key in var_dict if key not in ignore_ids} - - @staticmethod - def _convert_props_with_set(properties): - converted_properties = set() - for key, value in properties: - new_key = TemplateData.PROPS_CONVERSION[key] if key in \ - TemplateData.PROPS_CONVERSION else key - converted_properties.add((new_key, value)) - return converted_properties - - @staticmethod - def _convert_properties_with_dictionary(properties): - converted_properties = {} - for key, value in properties.items(): - new_key = TemplateData.PROPS_CONVERSION[key] if key in \ - TemplateData.PROPS_CONVERSION else key - converted_properties[new_key] = value - return converted_properties - - def _build_scenarios(self, scenarios_defs): - - scenarios = [] - for counter, scenario_def in enumerate(scenarios_defs): - scenario_id = "%s-scenario%s" % (self.name, str(counter)) - scenario_dict = scenario_def[TFields.SCENARIO] - scenarios.append(TemplateData.ScenarioData( - scenario_id, - scenario_dict, self).to_tuple()) - return scenarios - - class ScenarioData(object): - def __init__(self, scenario_id, scenario_dict, template_data): - self._template_entities = template_data.entities - self._template_relationships = template_data.relationships - - self._entities = {} - self._relationships = {} - - self.scenario_id = scenario_id - self.condition = parse_condition(scenario_dict[TFields.CONDITION]) - self.valid_target = self._calculate_missing_action_target() - self.actions = self._build_actions(scenario_dict[TFields.ACTIONS], - scenario_id) - self.subgraphs = TemplateData.SubGraph.from_condition( - self.condition, - self._extract_var_and_update_index) - - def __eq__(self, other): - return self.scenario_id == other.scenario_id \ - and self.condition == other.condition \ - and self.actions == other.actions - - def to_tuple(self): - return Scenario(id=self.scenario_id, - condition=self.condition, - actions=self.actions, - subgraphs=self.subgraphs, - entities=self._entities, - relationships=self._relationships) - - @classmethod - def build_equivalent_scenario(cls, - scenario, - template_id, - entity_props): - entities = scenario.entities.copy() - entities[template_id] = Vertex( - vertex_id=entities[template_id].vertex_id, - properties={k: v for k, v in entity_props}) - relationships = { - rel_id: cls.build_equivalent_relationship(rel, - template_id, - entity_props) - for rel_id, rel in scenario.relationships.items()} - - def extract_var(symbol_name): - if symbol_name in entities: - return entities[symbol_name], ENTITY - elif symbol_name in relationships: - return relationships[symbol_name], RELATIONSHIP - else: - raise VitrageError('invalid symbol name: {}' - .format(symbol_name)) - - subgraphs = TemplateData.SubGraph.from_condition( - scenario.condition, extract_var) - - return Scenario(id=scenario.id + '_equivalence', - condition=scenario.condition, - actions=scenario.actions, - subgraphs=subgraphs, - entities=entities, - relationships=relationships) - - @classmethod - def build_equivalent_relationship(cls, - relationship, - template_id, - entity_props): - source = relationship.source - target = relationship.target - if relationship.edge.source_id == template_id: - source = Vertex(vertex_id=source.vertex_id, - properties={k: v for k, v in entity_props}) - elif relationship.edge.target_id == template_id: - target = Vertex(vertex_id=target.vertex_id, - properties={k: v for k, v in entity_props}) - return EdgeDescription(source=source, - target=target, - edge=relationship.edge) - - def _build_actions(self, actions_def, scenario_id): - - actions = [] - for counter, action_def in enumerate(actions_def): - action_id = '%s-action%s' % (scenario_id, str(counter)) - action_dict = action_def[TFields.ACTION] - action_type = action_dict[TFields.ACTION_TYPE] - targets = action_dict.get(TFields.ACTION_TARGET, - self.valid_target) - properties = action_dict.get(TFields.PROPERTIES, {}) - - actions.append( - ActionSpecs(action_id, action_type, targets, properties)) - - return actions - - def _extract_var_and_update_index(self, symbol_name): - - if symbol_name in self._template_relationships: - relationship = self._template_relationships[symbol_name] - self._relationships[symbol_name] = relationship - self._entities.update({ - relationship.edge.source_id: relationship.source, - relationship.edge.target_id: relationship.target - }) - return relationship, RELATIONSHIP - - entity = self._template_entities[symbol_name] - self._entities[symbol_name] = entity - return entity, ENTITY - - def _calculate_missing_action_target(self): - """Return a vertex that can be used as an action target. - - External actions like execute_mistral do not have an explicit - action target. This parameter is a must for the sub-graph matching - algorithm. If it is missing, we would like to select an arbitrary - target from the condition. - - """ - definition_index = self._template_entities.copy() - definition_index.update(self._template_relationships) - targets = \ - get_condition_common_targets(self.condition, - definition_index, - self.TemplateDataSymbolResolver()) - return {TFields.TARGET: targets.pop()} if targets else None - - class TemplateDataSymbolResolver(SymbolResolver): - def is_relationship(self, symbol): - return isinstance(symbol, EdgeDescription) - - def get_relationship_source_id(self, relationship): - return relationship.source.vertex_id - - def get_relationship_target_id(self, relationship): - return relationship.target.vertex_id - - def get_entity_id(self, entity): - return entity.vertex_id - - class SubGraph(object): - @classmethod - def from_condition(cls, condition, extract_var): - return [cls.from_clause(clause, extract_var) - for clause in condition] - - @classmethod - def from_clause(cls, clause, extract_var): - condition_g = NXGraph("scenario condition") - - for term in clause: - variable, var_type = extract_var(term.symbol_name) - if var_type == ENTITY: - vertex = variable.copy() - vertex[VProps.VITRAGE_IS_DELETED] = False - vertex[VProps.VITRAGE_IS_PLACEHOLDER] = False - condition_g.add_vertex(vertex) - - else: # type = relationship - # prevent overwritten of NEG_CONDITION and - # VITRAGE_IS_DELETED property when there are both "not A" - # and "A" in same template - edge_desc = copy_edge_desc(variable) - cls._set_edge_relationship_info(edge_desc, term.positive) - cls._add_edge_relationship(condition_g, edge_desc) - - return condition_g - - @staticmethod - def _set_edge_relationship_info(edge_description, - is_positive_condition): - if not is_positive_condition: - edge_description.edge[NEG_CONDITION] = True - edge_description.edge[EProps.VITRAGE_IS_DELETED] = True - else: - edge_description.edge[EProps.VITRAGE_IS_DELETED] = False - edge_description.edge[NEG_CONDITION] = False - - edge_description.source[VProps.VITRAGE_IS_DELETED] = False - edge_description.source[VProps.VITRAGE_IS_PLACEHOLDER] = False - edge_description.target[VProps.VITRAGE_IS_DELETED] = False - edge_description.target[VProps.VITRAGE_IS_PLACEHOLDER] = False - - @staticmethod - def _add_edge_relationship(condition_graph, edge_description): - condition_graph.add_vertex(edge_description.source) - condition_graph.add_vertex(edge_description.target) - condition_graph.add_edge(edge_description.edge) diff --git a/vitrage/evaluator/template_loading/__init__.py b/vitrage/evaluator/template_loading/__init__.py new file mode 100644 index 000000000..6631c3f0f --- /dev/null +++ b/vitrage/evaluator/template_loading/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2017 - Alcatel-Lucent +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +__author__ = 'stack' diff --git a/vitrage/evaluator/equivalence_data.py b/vitrage/evaluator/template_loading/equivalence_loader.py similarity index 91% rename from vitrage/evaluator/equivalence_data.py rename to vitrage/evaluator/template_loading/equivalence_loader.py index bc7a6932d..3c3ebeac8 100644 --- a/vitrage/evaluator/equivalence_data.py +++ b/vitrage/evaluator/template_loading/equivalence_loader.py @@ -14,7 +14,7 @@ from vitrage.common.constants import TemplateTopologyFields from vitrage.common.exception import VitrageError -from vitrage.evaluator.template_data import TemplateData +from vitrage.evaluator.template_loading.props_converter import PropsConverter class Fields(TemplateTopologyFields): @@ -24,7 +24,7 @@ class Fields(TemplateTopologyFields): # noinspection PyAttributeOutsideInit -class EquivalenceData(object): +class EquivalenceLoader(object): def __init__(self, equivalence_def): @@ -54,7 +54,7 @@ class EquivalenceData(object): for k, v in entity_def[Fields.ENTITY].items()} entity_key = frozenset( - TemplateData._convert_props_with_set(entity_props)) + PropsConverter.convert_props_with_set(entity_props)) if entity_key in equivalence: raise VitrageError('duplicated entities found in ' 'equivalence') diff --git a/vitrage/evaluator/template_loading/props_converter.py b/vitrage/evaluator/template_loading/props_converter.py new file mode 100644 index 000000000..c18dda51d --- /dev/null +++ b/vitrage/evaluator/template_loading/props_converter.py @@ -0,0 +1,49 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.constants import VertexProperties as VProps + + +class PropsConverter(object): + + PROPS_CONVERSION = { + 'category': VProps.VITRAGE_CATEGORY, + 'type': VProps.VITRAGE_TYPE, + 'resource_id': VProps.VITRAGE_RESOURCE_ID, + 'sample_timestamp': VProps.VITRAGE_SAMPLE_TIMESTAMP, + 'is_deleted': VProps.VITRAGE_IS_DELETED, + 'is_placeholder': VProps.VITRAGE_IS_PLACEHOLDER, + 'aggregated_state': VProps.VITRAGE_AGGREGATED_STATE, + 'operational_state': VProps.VITRAGE_OPERATIONAL_STATE, + 'aggregated_severity': VProps.VITRAGE_AGGREGATED_SEVERITY, + 'operational_severity': VProps.VITRAGE_OPERATIONAL_SEVERITY + } + + @classmethod + def convert_props_with_set(cls, properties): + converted_properties = set() + for key, value in properties: + new_key = cls.PROPS_CONVERSION[key] if key in \ + cls.PROPS_CONVERSION else key + converted_properties.add((new_key, value)) + return converted_properties + + @classmethod + def convert_props_with_dictionary(cls, properties): + converted_properties = {} + for key, value in properties.items(): + new_key = cls.PROPS_CONVERSION[key] if key in \ + cls.PROPS_CONVERSION else key + converted_properties[new_key] = value + return converted_properties diff --git a/vitrage/evaluator/template_loading/scenario_loader.py b/vitrage/evaluator/template_loading/scenario_loader.py new file mode 100644 index 000000000..2b9e23474 --- /dev/null +++ b/vitrage/evaluator/template_loading/scenario_loader.py @@ -0,0 +1,165 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.exception import VitrageError +from vitrage.evaluator.condition import get_condition_common_targets +from vitrage.evaluator.condition import parse_condition +from vitrage.evaluator.condition import SymbolResolver +from vitrage.evaluator.template_data import ActionSpecs +from vitrage.evaluator.template_data import EdgeDescription +from vitrage.evaluator.template_data import ENTITY +from vitrage.evaluator.template_data import RELATIONSHIP +from vitrage.evaluator.template_data import Scenario +from vitrage.evaluator.template_fields import TemplateFields as TFields +from vitrage.evaluator.template_loading.subgraph_builder import SubGraphBuilder +from vitrage.graph import Vertex + + +class ScenarioLoader(object): + + def __init__(self, name, entities, relationships): + self.name = name + self._template_entities = entities + self._template_relationships = relationships + + self.entities = {} + self.relationships = {} + self.valid_target = None + + def build_scenarios(self, scenarios_defs): + scenarios = [] + for counter, scenario_def in enumerate(scenarios_defs): + scenario_id = "%s-scenario%s" % (self.name, str(counter)) + scenario_dict = scenario_def[TFields.SCENARIO] + condition = parse_condition(scenario_dict[TFields.CONDITION]) + self.valid_target = \ + self._calculate_missing_action_target(condition) + actions = self._build_actions(scenario_dict[TFields.ACTIONS], + scenario_id) + subgraphs = SubGraphBuilder.from_condition( + condition, + self._extract_var_and_update_index) + + scenarios.append( + Scenario(scenario_id, condition, actions, subgraphs, + self.entities, self.relationships)) + + return scenarios + + @classmethod + def build_equivalent_scenario(cls, scenario, template_id, entity_props): + entities = scenario.entities.copy() + entities[template_id] = Vertex( + vertex_id=entities[template_id].vertex_id, + properties={k: v for k, v in entity_props}) + relationships = { + rel_id: cls._build_equivalent_relationship(rel, + template_id, + entity_props) + for rel_id, rel in scenario.relationships.items()} + + def extract_var(symbol_name): + if symbol_name in entities: + return entities[symbol_name], ENTITY + elif symbol_name in relationships: + return relationships[symbol_name], RELATIONSHIP + else: + raise VitrageError('invalid symbol name: {}' + .format(symbol_name)) + + subgraphs = SubGraphBuilder.from_condition( + scenario.condition, extract_var) + + return Scenario(id=scenario.id + '_equivalence', + condition=scenario.condition, + actions=scenario.actions, + subgraphs=subgraphs, + entities=entities, + relationships=relationships) + + def _build_actions(self, actions_def, scenario_id): + actions = [] + + for counter, action_def in enumerate(actions_def): + action_id = '%s-action%s' % (scenario_id, str(counter)) + action_dict = action_def[TFields.ACTION] + action_type = action_dict[TFields.ACTION_TYPE] + targets = action_dict.get(TFields.ACTION_TARGET, + self.valid_target) + properties = action_dict.get(TFields.PROPERTIES, {}) + + actions.append( + ActionSpecs(action_id, action_type, targets, properties)) + + return actions + + def _extract_var_and_update_index(self, symbol_name): + if symbol_name in self._template_relationships: + relationship = self._template_relationships[symbol_name] + self.relationships[symbol_name] = relationship + self.entities.update({ + relationship.edge.source_id: relationship.source, + relationship.edge.target_id: relationship.target + }) + return relationship, RELATIONSHIP + + entity = self._template_entities[symbol_name] + self.entities[symbol_name] = entity + return entity, ENTITY + + def _calculate_missing_action_target(self, condition): + """Return a vertex that can be used as an action target. + + External actions like execute_mistral do not have an explicit + action target. This parameter is a must for the sub-graph matching + algorithm. If it is missing, we would like to select an arbitrary + target from the condition. + + """ + definition_index = self._template_entities.copy() + definition_index.update(self._template_relationships) + targets = \ + get_condition_common_targets(condition, + definition_index, + self.TemplateDataSymbolResolver()) + return {TFields.TARGET: targets.pop()} if targets else None + + class TemplateDataSymbolResolver(SymbolResolver): + def is_relationship(self, symbol): + return isinstance(symbol, EdgeDescription) + + def get_relationship_source_id(self, relationship): + return relationship.source.vertex_id + + def get_relationship_target_id(self, relationship): + return relationship.target.vertex_id + + def get_entity_id(self, entity): + return entity.vertex_id + + @staticmethod + def _build_equivalent_relationship(relationship, + template_id, + entity_props): + source = relationship.source + target = relationship.target + if relationship.edge.source_id == template_id: + source = Vertex(vertex_id=source.vertex_id, + properties={k: v for k, v in entity_props}) + elif relationship.edge.target_id == template_id: + target = Vertex(vertex_id=target.vertex_id, + properties={k: v for k, v in entity_props}) + return EdgeDescription(source=source, + target=target, + edge=relationship.edge) diff --git a/vitrage/evaluator/template_loading/subgraph_builder.py b/vitrage/evaluator/template_loading/subgraph_builder.py new file mode 100644 index 000000000..31d05d810 --- /dev/null +++ b/vitrage/evaluator/template_loading/subgraph_builder.py @@ -0,0 +1,76 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.constants import EdgeProperties as EProps +from vitrage.common.constants import VertexProperties as VProps +from vitrage.evaluator.template_data import EdgeDescription +from vitrage.evaluator.template_data import ENTITY +from vitrage.graph.algo_driver.sub_graph_matching import NEG_CONDITION +from vitrage.graph.driver.networkx_graph import NXGraph + + +class SubGraphBuilder(object): + @classmethod + def from_condition(cls, condition, extract_var): + return [cls.from_clause(clause, extract_var) + for clause in condition] + + @classmethod + def from_clause(cls, clause, extract_var): + condition_g = NXGraph("scenario condition") + + for term in clause: + variable, var_type = extract_var(term.symbol_name) + if var_type == ENTITY: + vertex = variable.copy() + vertex[VProps.VITRAGE_IS_DELETED] = False + vertex[VProps.VITRAGE_IS_PLACEHOLDER] = False + condition_g.add_vertex(vertex) + + else: # type = relationship + # prevent overwritten of NEG_CONDITION and + # VITRAGE_IS_DELETED property when there are both "not A" + # and "A" in same template + edge_desc = cls._copy_edge_desc(variable) + cls._set_edge_relationship_info(edge_desc, term.positive) + cls._add_edge_relationship(condition_g, edge_desc) + + return condition_g + + @staticmethod + def _set_edge_relationship_info(edge_description, + is_positive_condition): + if not is_positive_condition: + edge_description.edge[NEG_CONDITION] = True + edge_description.edge[EProps.VITRAGE_IS_DELETED] = True + else: + edge_description.edge[EProps.VITRAGE_IS_DELETED] = False + edge_description.edge[NEG_CONDITION] = False + + edge_description.source[VProps.VITRAGE_IS_DELETED] = False + edge_description.source[VProps.VITRAGE_IS_PLACEHOLDER] = False + edge_description.target[VProps.VITRAGE_IS_DELETED] = False + edge_description.target[VProps.VITRAGE_IS_PLACEHOLDER] = False + + @staticmethod + def _add_edge_relationship(condition_graph, edge_description): + condition_graph.add_vertex(edge_description.source) + condition_graph.add_vertex(edge_description.target) + condition_graph.add_edge(edge_description.edge) + + @staticmethod + def _copy_edge_desc(edge_desc): + return EdgeDescription(edge=edge_desc.edge.copy(), + source=edge_desc.source.copy(), + target=edge_desc.target.copy()) diff --git a/vitrage/evaluator/template_loading/template_loader.py b/vitrage/evaluator/template_loading/template_loader.py new file mode 100644 index 000000000..943884ec4 --- /dev/null +++ b/vitrage/evaluator/template_loading/template_loader.py @@ -0,0 +1,164 @@ +# Copyright 2016 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from vitrage.common.constants import VertexProperties as VProps +from vitrage.evaluator.template_data import EdgeDescription +from vitrage.evaluator.template_data import TemplateData +from vitrage.evaluator.template_fields import TemplateFields as TFields +from vitrage.evaluator.template_loading.props_converter import PropsConverter +from vitrage.evaluator.template_loading.scenario_loader import ScenarioLoader +from vitrage.graph import Edge +from vitrage.graph import Vertex +from vitrage.utils import evaluator as evaluator_utils + + +class TemplateLoader(object): + + PROPS_CONVERSION = { + 'category': VProps.VITRAGE_CATEGORY, + 'type': VProps.VITRAGE_TYPE, + 'resource_id': VProps.VITRAGE_RESOURCE_ID, + 'sample_timestamp': VProps.VITRAGE_SAMPLE_TIMESTAMP, + 'is_deleted': VProps.VITRAGE_IS_DELETED, + 'is_placeholder': VProps.VITRAGE_IS_PLACEHOLDER, + 'aggregated_state': VProps.VITRAGE_AGGREGATED_STATE, + 'operational_state': VProps.VITRAGE_OPERATIONAL_STATE, + 'aggregated_severity': VProps.VITRAGE_AGGREGATED_SEVERITY, + 'operational_severity': VProps.VITRAGE_OPERATIONAL_SEVERITY + } + + def __init__(self): + self.entities = {} + self.relationships = {} + + def load(self, template_def, def_templates=None): + name = template_def[TFields.METADATA][TFields.NAME] + + if def_templates is None: + def_templates = {} + defs = {} + + if TFields.DEFINITIONS in template_def: + defs = template_def[TFields.DEFINITIONS] + if TFields.ENTITIES in defs: + self.entities = self._build_entities(defs[TFields.ENTITIES]) + + # Add definitions from template then from definition templates. + if TFields.INCLUDES in template_def: + includes = template_def[TFields.INCLUDES] + self._build_entities_from_def_templates( + includes, def_templates, self.entities) + + if TFields.RELATIONSHIPS in defs: + self.relationships = self._build_relationships( + defs[TFields.RELATIONSHIPS]) + + if TFields.INCLUDES in template_def: + includes = template_def[TFields.INCLUDES] + self._build_relationships_with_def_templates(includes, + def_templates, + self.relationships) + + scenarios = ScenarioLoader(name, self.entities, self.relationships)\ + .build_scenarios(template_def[TFields.SCENARIOS]) + + return TemplateData(name, self.entities, self.relationships, scenarios) + + def _build_entities(self, entities_defs): + entities = {} + for entity_def in entities_defs: + + entity_dict = entity_def[TFields.ENTITY] + template_id = entity_dict[TFields.TEMPLATE_ID] + properties = PropsConverter.convert_props_with_dictionary( + self._extract_properties(entity_dict)) + entities[template_id] = Vertex(template_id, properties) + + return entities + + def _build_entities_from_def_templates( + self, includes, def_templates, entities): + + for def_template_dict in includes: + + name = def_template_dict[TFields.NAME] + def_template = evaluator_utils.find_def_template( + name, def_templates) + defs = def_template[TFields.DEFINITIONS] + entities_defs = defs[TFields.ENTITIES] + + for entity_def in entities_defs: + + entity_dict = entity_def[TFields.ENTITY] + template_id = entity_dict[TFields.TEMPLATE_ID] + if template_id not in entities: + + properties = \ + PropsConverter.convert_props_with_dictionary( + self._extract_properties(entity_dict)) + entities[template_id] = Vertex(template_id, properties) + + def _build_relationships(self, relationships_defs): + + relationships = {} + for relationship_def in relationships_defs: + + relationship_dict = relationship_def[TFields.RELATIONSHIP] + relationship = self._extract_relationship_info(relationship_dict) + template_id = relationship_dict[TFields.TEMPLATE_ID] + relationships[template_id] = relationship + + return relationships + + def _build_relationships_with_def_templates( + self, includes, def_templates, relationships): + + for def_template_dict in includes: + + name = def_template_dict[TFields.NAME] + def_template = evaluator_utils.find_def_template( + name, def_templates) + + if TFields.RELATIONSHIPS in def_template[TFields.DEFINITIONS]: + defs = def_template[TFields.DEFINITIONS] + relationship_defs = defs[TFields.RELATIONSHIPS] + + for relationship_def in relationship_defs: + relationship_dict = relationship_def[TFields.RELATIONSHIP] + template_id = relationship_dict[TFields.TEMPLATE_ID] + + if template_id not in relationships: + relationship = self._extract_relationship_info( + relationship_dict) + relationships[template_id] = relationship + + def _extract_relationship_info(self, relationship_dict): + source_id = relationship_dict[TFields.SOURCE] + target_id = relationship_dict[TFields.TARGET] + + edge = Edge(source_id, + target_id, + relationship_dict[TFields.RELATIONSHIP_TYPE], + self._extract_properties(relationship_dict)) + + source = self.entities[source_id] + target = self.entities[target_id] + return EdgeDescription(edge, source, target) + + @staticmethod + def _extract_properties(var_dict): + + ignore_ids = [TFields.TEMPLATE_ID, TFields.SOURCE, TFields.TARGET] + return \ + {key: var_dict[key] for key in var_dict if key not in ignore_ids} diff --git a/vitrage/tests/unit/evaluator/test_condition.py b/vitrage/tests/unit/evaluator/test_condition.py index 032bac243..a7e2ad4c0 100644 --- a/vitrage/tests/unit/evaluator/test_condition.py +++ b/vitrage/tests/unit/evaluator/test_condition.py @@ -12,9 +12,9 @@ # License for the specific language governing permissions and limitations # under the License. -from vitrage.evaluator.condition import EdgeDescription from vitrage.evaluator.condition import SymbolResolver -from vitrage.evaluator.template_data import TemplateData +from vitrage.evaluator.template_data import EdgeDescription +from vitrage.evaluator.template_loading.template_loader import TemplateLoader from vitrage.evaluator.template_validation.content.scenario_validator \ import get_condition_common_targets from vitrage.tests import base @@ -93,7 +93,7 @@ class ConditionTest(base.BaseTest): template_name) template_definition = file_utils.load_yaml_file(template_path, True) - template_data = TemplateData(template_definition) + template_data = TemplateLoader().load(template_definition) definitions_index = template_data.entities.copy() definitions_index.update(template_data.relationships) diff --git a/vitrage/tests/unit/evaluator/test_equivalence_data.py b/vitrage/tests/unit/evaluator/test_equivalence_loader.py similarity index 92% rename from vitrage/tests/unit/evaluator/test_equivalence_data.py rename to vitrage/tests/unit/evaluator/test_equivalence_loader.py index 168a5e037..0bac1c23b 100644 --- a/vitrage/tests/unit/evaluator/test_equivalence_data.py +++ b/vitrage/tests/unit/evaluator/test_equivalence_loader.py @@ -18,7 +18,8 @@ from vitrage.datasources.nagios import NAGIOS_DATASOURCE from vitrage.datasources.zabbix import ZABBIX_DATASOURCE from vitrage.evaluator.actions.evaluator_event_transformer \ import VITRAGE_DATASOURCE -from vitrage.evaluator.equivalence_data import EquivalenceData +from vitrage.evaluator.template_loading.equivalence_loader import \ + EquivalenceLoader from vitrage.tests import base from vitrage.tests.mocks import utils from vitrage.utils import file as file_utils @@ -35,8 +36,7 @@ class EquivalenceTemplateTest(base.BaseTest): self.BASIC_TEMPLATE) equivalence_definition = file_utils.load_yaml_file(equivalence_path, True) - equivalence_data = EquivalenceData(equivalence_definition) - equivalences = equivalence_data.equivalences + equivalences = EquivalenceLoader(equivalence_definition).equivalences expected = [ frozenset([ diff --git a/vitrage/tests/unit/evaluator/test_template_data.py b/vitrage/tests/unit/evaluator/test_template_loader.py similarity index 96% rename from vitrage/tests/unit/evaluator/test_template_data.py rename to vitrage/tests/unit/evaluator/test_template_loader.py index 8cb86e190..d1548a54a 100644 --- a/vitrage/tests/unit/evaluator/test_template_data.py +++ b/vitrage/tests/unit/evaluator/test_template_loader.py @@ -25,8 +25,8 @@ from vitrage.evaluator.scenario_evaluator import ActionType from vitrage.evaluator.template_data import ActionSpecs from vitrage.evaluator.template_data import EdgeDescription from vitrage.evaluator.template_data import Scenario -from vitrage.evaluator.template_data import TemplateData from vitrage.evaluator.template_fields import TemplateFields as TFields +from vitrage.evaluator.template_loading.template_loader import TemplateLoader from vitrage.graph import Edge from vitrage.graph import Vertex from vitrage.tests import base @@ -53,7 +53,8 @@ class BasicTemplateTest(base.BaseTest): def_templates_path) def_templates_dict = utils.get_def_templates_dict_from_list( def_demplates_list) - template_data = TemplateData(template_definition, def_templates_dict) + template_data = \ + TemplateLoader().load(template_definition, def_templates_dict) entities = template_data.entities relationships = template_data.relationships scenarios = template_data.scenarios @@ -70,8 +71,8 @@ class BasicTemplateTest(base.BaseTest): # Assertions for definition in definitions[TFields.ENTITIES]: for key, value in definition['entity'].items(): - new_key = TemplateData.PROPS_CONVERSION[key] if key in \ - TemplateData.PROPS_CONVERSION else key + new_key = TemplateLoader.PROPS_CONVERSION[key] if key in \ + TemplateLoader.PROPS_CONVERSION else key del definition['entity'][key] definition['entity'][new_key] = value self._validate_entities(entities, definitions[TFields.ENTITIES]) @@ -159,7 +160,7 @@ class BasicTemplateTest(base.BaseTest): self.BASIC_TEMPLATE) template_definition = file_utils.load_yaml_file(template_path, True) - template_data = TemplateData(template_definition) + template_data = TemplateLoader().load(template_definition) entities = template_data.entities relationships = template_data.relationships scenarios = template_data.scenarios @@ -168,8 +169,8 @@ class BasicTemplateTest(base.BaseTest): # Assertions for definition in definitions[TFields.ENTITIES]: for key, value in definition['entity'].items(): - new_key = TemplateData.PROPS_CONVERSION[key] if key in \ - TemplateData.PROPS_CONVERSION else key + new_key = TemplateLoader.PROPS_CONVERSION[key] if key in \ + TemplateLoader.PROPS_CONVERSION else key del definition['entity'][key] definition['entity'][new_key] = value self._validate_entities(entities, definitions[TFields.ENTITIES])