Added support and documentation for regular expressions in template entity definitions

Implements: blueprint entity-regex-definition

Change-Id: I5ac81912a88d9a38dc08b23d93e79042f0c62200

doc/source/contributor/vitrage-template-format.rst

@@ -50,6 +50,7 @@ The template is divided into four main sections:
    - *condition –* the condition to be met. This condition will be phrased referencing the entities and relationships previously defined.
    - *action(s) –* a list of actions to execute when the condition is met.
 
+
 Definition Template Structure
 -----------------------------
 These are separate files, which contain only definitions and can be included under the includes section in regular templates. The definition templates are written in YAML
@@ -378,6 +379,21 @@ Common parameters and their acceptable values - for writing templates
 |                   |                       | mark_down               |                                    |
 +-------------------+-----------------------+-------------------------+------------------------------------+
 
+Using regular expressions in an entity definition
+-------------------------------------------------
+All parameters within an entity defintion can be made to include regular
+expressions. To do this, simply add ".regex" to their key. For example, as
+Zabbix supports regular expressions and a Zabbix alarm contains a "rawtext"
+field which is a regular expression, a Zabbix alarm entity defined in the
+template may contain a "rawtext.regex" field that is also defined by a
+regular expression:
+::
+
+  - entity:
+     category: ALARM
+     type: zabbix
+     rawtext.regex: Interface ([_a-zA-Z0-9'-]+) down on {HOST.NAME}
+     template_id: zabbix_alarm
 
 Supported Actions
 -----------------
@@ -396,7 +412,7 @@ Raise a deduced alarm on a target entity
                 target: instance         # mandatory. entity (from the definitions section) to raise an alarm on. Should not be an alarm.
 
 set_state
-^^^^^^^^^^^
+^^^^^^^^^
 Set state of specified entity. This will directly affect the state as seen in vitrage, but will not impact the state at the relevant datasource of this entity.
  ::
 

vitrage/evaluator/scenario_repository.py

@@ -33,6 +33,7 @@ from vitrage.evaluator.template_validation.template_syntax_validator import \
     EXCEPTION
 from vitrage.evaluator.template_validation.template_syntax_validator import \
     syntax_validation
+from vitrage.graph.filter import check_filter as check_subset
 from vitrage.utils import datetime as datetime_utils
 from vitrage.utils import file as file_utils
 
@@ -72,15 +73,11 @@ class ScenarioRepository(object):
 
     def get_scenarios_by_vertex(self, vertex):
 
-        try:
-            entity_key = frozenset(vertex.properties.items())
-        except Exception as e:
-            LOG.error('frozenset for vertex failed %s', str(vertex))
-            raise e
+        entity_key = vertex.properties
 
         scenarios = []
         for scenario_key, value in self.entity_scenarios.items():
-            if scenario_key.issubset(entity_key):
+            if check_subset(entity_key, dict(scenario_key)):
                 scenarios += value
         return scenarios
 
@@ -92,10 +89,11 @@ class ScenarioRepository(object):
         for scenario_key, value in self.relationship_scenarios.items():
 
             check_label = key.label == scenario_key.label
-            check_source_issubset = scenario_key.source.issubset(key.source)
-            check_target_issubset = scenario_key.target.issubset(key.target)
-
-            if check_label and check_source_issubset and check_target_issubset:
+            if check_label \
+                    and check_subset(dict(key.source),
+                                     dict(scenario_key.source)) \
+                    and check_subset(dict(key.target),
+                                     dict(scenario_key.target)):
                 scenarios += value
 
         return scenarios

vitrage/evaluator/template_fields.py

@@ -33,3 +33,4 @@ class TemplateFields(TemplateTopologyFields):
     TEMPLATE_ID = 'template_id'
 
     PROPERTIES = 'properties'
+    REGEX = '.regex'

vitrage/evaluator/template_validation/content/template_content_validator.py

@@ -16,6 +16,7 @@ from sympy.logic.boolalg import Not
 from sympy import Symbol
 
 from oslo_log import log
+import re
 from six.moves import reduce
 
 from vitrage.common.constants import EdgeProperties as EProps
@@ -138,6 +139,16 @@ def _validate_entity_definition(entity_dict, entities_index):
         LOG.error('%s status code: %s' % (status_msgs[2], 2))
         return get_content_fault_result(2)
 
+    for key, value in entity_dict.items():
+
+        if key.lower().endswith(TemplateFields.REGEX):
+            try:
+                re.compile(value)
+            except Exception:
+                LOG.error('%s %s status code: %s' % (status_msgs[47],
+                                                     str(key), 47))
+                return get_content_fault_result(47)
+
     return get_content_correct_result()
 
 

vitrage/evaluator/template_validation/status_messages.py

@@ -36,6 +36,7 @@ status_msgs = {
     45: 'Invalid entity category. Category must be from types: '
         '{categories}'.format(categories=EntityCategory.categories()),
     46: 'Entity field is required.',
+    47: 'Invalid regular expression defined in field:',
 
     # metadata section status messages 60-79
     60: 'metadata section must contain id field.',

vitrage/graph/filter.py

@@ -12,6 +12,9 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+from vitrage.evaluator.template_fields import TemplateFields as Fields
+from vitrage.graph.utils import check_property_with_regex
+
 
 def check_filter(data, attr_filter, *args):
     """Check attr_filter against data
@@ -20,7 +23,7 @@ def check_filter(data, attr_filter, *args):
     :param attr_filter: a dictionary of either
     field_name : value (mandatory)
     field_name : list of values - data[field_name] must match ANY of the values
-    :param args: list of filter keys to ignore (if exist)
+    :param args: list of filter keys to ignore  (if exist)
     :rtype: bool
     """
     if not attr_filter:
@@ -30,6 +33,12 @@ def check_filter(data, attr_filter, *args):
             continue
         if not isinstance(content, list):
             content = [content]
-        if not data.get(key) in content:
-            return False
+        if data.get(key) not in content:
+            if key.lower().endswith(Fields.REGEX):
+                new_key = key[:-len(Fields.REGEX)]
+                if not check_property_with_regex(new_key, content[0],
+                                                 data):
+                    return False
+            else:
+                return False
     return True

vitrage/graph/utils.py

@@ -12,6 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import re
 from vitrage.common.constants import EdgeProperties as EConst
 from vitrage.common.constants import VertexProperties as VConst
 from vitrage.graph.driver.elements import Edge
@@ -113,3 +114,20 @@ def create_edge(source_id,
                 label=relationship_type,
                 properties=properties)
     return edge
+
+
+def check_property_with_regex(key, regex, data):
+    """Checks if the contents of data[key] matches the given regex
+
+    :param: key: key to find in data
+    :param: data: dict to search
+    :type: data: dict
+    :param: regex: regular expression to check against
+    :type: regex: str
+    :rtype: bool
+    """
+    value = data.get(key)
+    if value is None:
+        return False
+    pattern = re.compile(regex)
+    return pattern.match(value) is not None

vitrage/tests/resources/templates/regex/basic_regex.yaml

@@ -0,0 +1,32 @@
+metadata:
+ name: basic_regex
+ description: a basic template with a regex in the "rawtext" field
+definitions:
+ entities:
+  - entity:
+     category: ALARM
+     type: zabbix
+     rawtext.regex: Interface ([_a-zA-Z0-9'-]+) down on {HOST.NAME}
+     template_id: zabbix_alarm_pass
+  - entity:
+     category: RESOURCE
+     type: nova.host
+     name: host-$1
+     template_id: host
+ relationships:
+  - relationship:
+     source: zabbix_alarm_pass
+     relationship_type: on
+     target: host
+     template_id : nic_fail_on_host
+scenarios:
+ - scenario:
+    condition: nic_fail_on_host
+    actions:
+     - action:
+        action_type: raise_alarm
+        action_target:
+         target: host
+        properties:
+         alarm_name: nic problem
+         severity: critical

vitrage/tests/resources/templates/regex/basic_regex_for_fail.yaml

@@ -0,0 +1,32 @@
+metadata:
+ name: basic_regex_for_fail
+ description: a basic template with a regex for failing tests
+definitions:
+ entities:
+  - entity:
+     category: ALARM
+     type: zabbix
+     rawtext.regex: TEST TEXT fail
+     template_id: zabbix_alarm
+  - entity:
+     category: RESOURCE
+     type: nova.host
+     name: host-$1
+     template_id: host
+ relationships:
+  - relationship:
+     source: zabbix_alarm
+     relationship_type: on
+     target: host
+     template_id : nic_fail_on_host
+scenarios:
+ - scenario:
+    condition: nic_fail_on_host
+    actions:
+     - action:
+        action_type: raise_alarm
+        action_target:
+         target: host
+        properties:
+         alarm_name: nic problem
+         severity: critical

vitrage/tests/resources/templates/regex/faulty_regex.yaml

@@ -0,0 +1,32 @@
+metadata:
+ name: faulty_regex
+ description: a basic template with a badly defined regex
+definitions:
+ entities:
+  - entity:
+     category: ALARM
+     type: zabbix
+     rawtext.regex: \w+ c(
+     template_id: zabbix_alarm_pass
+  - entity:
+     category: RESOURCE
+     type: nova.host
+     name: host-$1
+     template_id: host
+ relationships:
+  - relationship:
+     source: zabbix_alarm_pass
+     relationship_type: on
+     target: host
+     template_id : nic_fail_on_host
+scenarios:
+ - scenario:
+    condition: nic_fail_on_host
+    actions:
+     - action:
+        action_type: raise_alarm
+        action_target:
+         target: host
+        properties:
+         alarm_name: nic problem
+         severity: critical

vitrage/tests/resources/templates/regex/regex_for_exact_match.yaml

@@ -0,0 +1,32 @@
+metadata:
+ name: regex_for_exact_match
+ description: a template with a regex for an exact match
+definitions:
+ entities:
+  - entity:
+     category: ALARM
+     type: zabbix
+     rawtext.regex: Public interface host43 down
+     template_id: exact_match
+  - entity:
+     category: RESOURCE
+     type: nova.host
+     name: host-$1
+     template_id: host
+ relationships:
+  - relationship:
+     source: exact_match
+     relationship_type: on
+     target: host
+     template_id : nic_fail_on_host
+scenarios:
+ - scenario:
+    condition: nic_fail_on_host
+    actions:
+     - action:
+        action_type: raise_alarm
+        action_target:
+         target: host
+        properties:
+         alarm_name: nic problem
+         severity: critical

vitrage/tests/unit/evaluator/template_validation/content/test_template_content_validator.py

@@ -24,6 +24,7 @@ from vitrage.utils import file as file_utils
 
 
 CONDITION_TEMPLATES_DIR = '%s/templates/evaluator/conditions/%s'
+REGEX_TEMPLATE_DIR = '%s/templates/regex/%s'
 
 
 class TemplateContentValidatorTest(ValidatorTest):
@@ -258,6 +259,24 @@ class TemplateContentValidatorTest(ValidatorTest):
         self._execute_condition_template_with_fault_result(
             'complex_not_unsupported.yaml', 135)
 
+    def test_faulty_regex(self):
+        faulty_regex_path = \
+            REGEX_TEMPLATE_DIR % (utils.get_resources_dir(),
+                                  "faulty_regex.yaml")
+        faulty_regex_template = \
+            file_utils.load_yaml_file(faulty_regex_path)
+        self._execute_and_assert_with_fault_result(
+            faulty_regex_template, 47)
+
+    def test_basic_regex(self):
+        basic_regex_path = \
+            REGEX_TEMPLATE_DIR % (utils.get_resources_dir(),
+                                  "basic_regex.yaml")
+        basic_regex_template = \
+            file_utils.load_yaml_file(basic_regex_path)
+        self._execute_and_assert_with_correct_result(
+            basic_regex_template)
+
     def _execute_condition_template_with_correct_result(self, template_name):
         template_path = CONDITION_TEMPLATES_DIR % (utils.get_resources_dir(),
                                                    template_name)

vitrage/tests/unit/evaluator/test_scenario_repository.py

@@ -21,6 +21,7 @@ from vitrage.common.constants import VertexProperties as VProps
 from vitrage.evaluator.scenario_repository import ScenarioRepository
 from vitrage.evaluator.template_validation.template_syntax_validator import \
     syntax_validation
+from vitrage.graph import Vertex
 from vitrage.tests import base
 from vitrage.tests.mocks import utils
 from vitrage.utils import file as file_utils
@@ -99,6 +100,75 @@ class ScenarioRepositoryTest(base.BaseTest):
         pass
 
 
+class RegExTemplateTest(base.BaseTest):
+
+    BASE_DIR = utils.get_resources_dir() + '/templates/regex'
+    OPTS = [
+        cfg.StrOpt('templates_dir',
+                   default=BASE_DIR),
+        cfg.StrOpt('equivalences_dir',
+                   default=BASE_DIR + '/equivalences')]
+
+    @classmethod
+    def setUpClass(cls):
+
+        cls.conf = cfg.ConfigOpts()
+        cls.conf.register_opts(cls.OPTS, group='evaluator')
+        cls.scenario_repository = ScenarioRepository(cls.conf)
+
+    def test_basic_regex(self):
+
+        event_properties = {
+            "time": 121354,
+            "vitrage_type": "zabbix",
+            "vitrage_category": "ALARM",
+            "rawtext": "Interface virtual-0 down on {HOST.NAME}",
+            "host": "some_host_kukoo"
+        }
+        event_vertex = Vertex(vertex_id="test_vertex",
+                              properties=event_properties)
+        relevant_scenarios = \
+            self.scenario_repository.get_scenarios_by_vertex(
+                event_vertex)
+        self.assertEqual(1, len(relevant_scenarios))
+        relevant_scenario = relevant_scenarios[0]
+        self.assertEqual("zabbix_alarm_pass", relevant_scenario[0].vertex_id)
+
+    def test_regex_with_exact_match(self):
+
+        event_properties = {
+            "time": 121354,
+            "vitrage_type": "zabbix",
+            "vitrage_category": "ALARM",
+            "rawtext": "Public interface host43 down",
+            "host": "some_host_kukoo"
+        }
+        event_vertex = Vertex(vertex_id="test_vertex",
+                              properties=event_properties)
+        relevant_scenarios = \
+            self.scenario_repository.get_scenarios_by_vertex(
+                event_vertex)
+        self.assertEqual(1, len(relevant_scenarios))
+        relevant_scenario = relevant_scenarios[0]
+        self.assertEqual("exact_match", relevant_scenario[0].vertex_id)
+
+    def test_basic_regex_with_no_match(self):
+
+        event_properties = {
+            "time": 121354,
+            "vitrage_type": "zabbix",
+            "vitrage_category": "ALARM",
+            "rawtext": "No Match",
+            "host": "some_host_kukoo"
+        }
+        event_vertex = Vertex(vertex_id="test_vertex",
+                              properties=event_properties)
+        relevant_scenarios = \
+            self.scenario_repository.get_scenarios_by_vertex(
+                event_vertex)
+        self.assertEqual(0, len(relevant_scenarios))
+
+
 class EquivalentScenarioTest(base.BaseTest):
     BASE_DIR = utils.get_resources_dir() + '/templates/equivalent_scenarios/'
     OPTS = [

vitrage/tests/unit/graph/test_graph.py

@@ -22,6 +22,7 @@ Tests for `vitrage` graph driver
 from vitrage.common.constants import EdgeProperties as EProps
 from vitrage.common.constants import VertexProperties as VProps
 from vitrage.graph import Direction
+from vitrage.graph.filter import check_filter
 from vitrage.graph import utils
 from vitrage.tests.unit.graph.base import *  # noqa
 
@@ -529,3 +530,42 @@ class TestGraph(GraphTestBase):
 
         e = g1.get_vertex(v3.vertex_id)
         self.assertIsNotNone(e, 'Vertex missing after graphs union')
+
+
+class TestFilter(base.BaseTest):
+
+    def test_basic_regex(self):
+        event_properties = {
+            "time": 121354,
+            "vitrage_type": "zabbix",
+            "vitrage_category": "ALARM",
+            "rawtext": "Interface kukoo down on {HOST.NAME}",
+            "host": "some_host_kukoo"
+        }
+
+        attr_filter = {
+            "vitrage_category": "ALARM",
+            "rawtext.regex": "Interface ([_a-zA-Z0-9'\-]+) down on {"
+                             "HOST.NAME}",
+            "host": "some_host_kukoo"
+        }
+        self.assertEqual(True, check_filter(data=event_properties,
+                                            attr_filter=attr_filter))
+
+    def test_basic_regex_with_no_match(self):
+        event_properties = {
+            "time": 121354,
+            "vitrage_type": "zabbix",
+            "vitrage_category": "ALARM",
+            "rawtext": "Text With No Match",
+            "host": "some_host_kukoo"
+        }
+
+        attr_filter = {
+            "vitrage_category": "ALARM",
+            "rawtext.RegEx": "Interface ([_a-zA-Z0-9'\-]+) down on {"
+                             "HOST.NAME}",
+            "host": "some_host_kukoo"
+        }
+        self.assertEqual(False, check_filter(data=event_properties,
+                                             attr_filter=attr_filter))