From 9c47a1178e93bc03643279748098f9e97dbbb3a2 Mon Sep 17 00:00:00 2001 From: Muhamad Najjar Date: Mon, 1 Jan 2018 13:58:34 +0000 Subject: [PATCH] Graph Persistor Store/Load graph snapshots in/from database Part from Vitrage HA and History Vision https://docs.openstack.org/vitrage/latest/contributor/vitrage-ha-and-history-vision.html Change-Id: I92ca74dabc22e8991c96d7f090be9978b8c93894 --- devstack/gate_hook.sh | 4 +- vitrage/cli/persistor.py | 2 +- vitrage/common/exception.py | 4 + vitrage/datasources/collector_notifier.py | 4 +- vitrage/entity_graph/processor/processor.py | 7 +- vitrage/entity_graph/service.py | 14 +- vitrage/graph/driver/networkx_graph.py | 9 + vitrage/opts.py | 4 +- .../{persistor => persistency}/__init__.py | 8 +- vitrage/persistency/graph_persistor.py | 59 ++++++ vitrage/{persistor => persistency}/service.py | 2 +- vitrage/storage/base.py | 33 +++- vitrage/storage/impl_sqlalchemy.py | 64 ++++++- vitrage/storage/sqlalchemy/models.py | 17 ++ vitrage/tests/base.py | 23 +++ .../entity_graph/graph_persistor/__init__.py | 15 ++ .../graph_persistor/test_graph_persistor.py | 94 ++++++++++ vitrage/tests/mocks/graph_generator.py | 169 ++++++++++++++++++ .../driver/driver_stack_snapshot_dynamic.json | 2 +- .../mock_configurations/edges/attached.json | 5 + .../mock_configurations/edges/contains.json | 5 + .../mock_configurations/edges/on.json | 5 + .../vertices/cinder.volume.json | 19 ++ .../vertices/neutron.network.json | 15 ++ .../vertices/neutron.port.json | 18 ++ .../vertices/nova.host.json | 11 ++ .../vertices/nova.instance.json | 17 ++ .../vertices/nova.zone.json | 14 ++ .../vertices/openstack-cluster.json | 13 ++ .../vertices/tripleo.controller.json | 13 ++ .../vertices/vitrage.alarm.json | 18 ++ .../mock_configurations/vertices/zabbix.json | 18 ++ .../tests/api/rca/test_rca.py | 4 +- .../tests/api/templates/test_template.py | 4 +- 34 files changed, 690 insertions(+), 23 deletions(-) rename vitrage/{persistor => persistency}/__init__.py (74%) create mode 100644 vitrage/persistency/graph_persistor.py rename vitrage/{persistor => persistency}/service.py (97%) create mode 100644 vitrage/tests/functional/entity_graph/graph_persistor/__init__.py create mode 100644 vitrage/tests/functional/entity_graph/graph_persistor/test_graph_persistor.py create mode 100644 vitrage/tests/mocks/graph_generator.py create mode 100644 vitrage/tests/resources/mock_configurations/edges/attached.json create mode 100644 vitrage/tests/resources/mock_configurations/edges/contains.json create mode 100644 vitrage/tests/resources/mock_configurations/edges/on.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/cinder.volume.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/neutron.network.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/neutron.port.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/nova.host.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/nova.instance.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/nova.zone.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/openstack-cluster.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/tripleo.controller.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/vitrage.alarm.json create mode 100644 vitrage/tests/resources/mock_configurations/vertices/zabbix.json diff --git a/devstack/gate_hook.sh b/devstack/gate_hook.sh index dfdac0614..00a65a431 100644 --- a/devstack/gate_hook.sh +++ b/devstack/gate_hook.sh @@ -74,8 +74,8 @@ changes_interval = 5 [datasources] snapshots_interval = 120 -[persistor] -persist_events=true +[persistency] +enable_persistency=true EOF )" diff --git a/vitrage/cli/persistor.py b/vitrage/cli/persistor.py index c22f278e7..62564be76 100644 --- a/vitrage/cli/persistor.py +++ b/vitrage/cli/persistor.py @@ -17,7 +17,7 @@ import sys from oslo_log import log from oslo_service import service as os_service from vitrage.cli import VITRAGE_TITLE -from vitrage.persistor.service import PersistorService +from vitrage.persistency.service import PersistorService from vitrage import service from vitrage import storage diff --git a/vitrage/common/exception.py b/vitrage/common/exception.py index 9b29f1ed3..2354a35f7 100644 --- a/vitrage/common/exception.py +++ b/vitrage/common/exception.py @@ -30,6 +30,10 @@ class VitrageError(VitrageException): """Exception for a serious error in Vitrage""" +class VitrageInputError(VitrageError): + """Exception raised for errors in the input""" + + class VitrageAlgorithmError(VitrageException): """Exception for unexpected termination of algorithms.""" diff --git a/vitrage/datasources/collector_notifier.py b/vitrage/datasources/collector_notifier.py index 3cb99fc17..54da41429 100644 --- a/vitrage/datasources/collector_notifier.py +++ b/vitrage/datasources/collector_notifier.py @@ -27,8 +27,8 @@ class CollectorNotifier(object): self.oslo_notifier = None try: topics = [conf.datasources.notification_topic_collector] - if conf.persistor.persist_events: - topics.append(conf.persistor.persistor_topic) + if conf.persistency.enable_persistency: + topics.append(conf.persistency.persistor_topic) else: LOG.warning("Not persisting events") diff --git a/vitrage/entity_graph/processor/processor.py b/vitrage/entity_graph/processor/processor.py index 036376d49..041d0204a 100644 --- a/vitrage/entity_graph/processor/processor.py +++ b/vitrage/entity_graph/processor/processor.py @@ -12,7 +12,6 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - from oslo_log import log from vitrage.common.constants import EntityCategory @@ -32,7 +31,8 @@ LOG = log.getLogger(__name__) class Processor(processor.ProcessorBase): - def __init__(self, conf, initialization_status, e_graph): + def __init__(self, conf, initialization_status, e_graph, + graph_persistor=None): super(Processor, self).__init__() self.conf = conf self.transformer_manager = TransformerManager(self.conf) @@ -41,6 +41,7 @@ class Processor(processor.ProcessorBase): self.initialization_status = initialization_status self.entity_graph = e_graph self._notifier = GraphNotifier(conf) + self._graph_persistor = graph_persistor def process_event(self, event): """Decides which action to run on given event @@ -59,6 +60,8 @@ class Processor(processor.ProcessorBase): entity = self.transformer_manager.transform(event) self._calculate_vitrage_aggregated_state(entity.vertex, entity.action) self.actions[entity.action](entity.vertex, entity.neighbors) + if self._graph_persistor: + self._graph_persistor.update_last_event_timestamp(event) def create_entity(self, new_vertex, neighbors): """Adds new vertex to the entity graph diff --git a/vitrage/entity_graph/service.py b/vitrage/entity_graph/service.py index f37a3fc9a..2effedcfe 100644 --- a/vitrage/entity_graph/service.py +++ b/vitrage/entity_graph/service.py @@ -11,11 +11,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import oslo_messaging import threading import time from oslo_log import log -import oslo_messaging from oslo_service import service as os_service from vitrage.entity_graph import EVALUATOR_TOPIC @@ -23,6 +23,7 @@ from vitrage.entity_graph.processor.processor import Processor from vitrage.entity_graph.vitrage_init import VitrageInit from vitrage.evaluator.evaluator_service import EvaluatorManager from vitrage import messaging +from vitrage.persistency.graph_persistor import GraphPersistor LOG = log.getLogger(__name__) @@ -37,7 +38,10 @@ class VitrageGraphService(os_service.Service): self.graph = graph self.evaluator = EvaluatorManager(conf, graph) self.init = VitrageInit(conf, graph, self.evaluator) - self.processor = Processor(self.conf, self.init, e_graph=graph) + self.graph_persistor = GraphPersistor(conf) if \ + self.conf.persistency.enable_persistency else None + self.processor = Processor(self.conf, self.init, graph, + self.graph_persistor) self.listener = self._init_listener() def _init_listener(self): @@ -52,6 +56,12 @@ class VitrageGraphService(os_service.Service): def start(self): LOG.info("Vitrage Graph Service - Starting...") super(VitrageGraphService, self).start() + if self.graph_persistor: + self.tg.add_timer( + self.conf.persistency.graph_persistency_interval, + self.graph_persistor.store_graph, + self.conf.persistency.graph_persistency_interval, + graph=self.graph) self.tg.add_thread( self.init.initializing_process, on_end_messages_func=self.processor.on_recieved_all_end_messages) diff --git a/vitrage/graph/driver/networkx_graph.py b/vitrage/graph/driver/networkx_graph.py index f6c6eac66..0c7c816a9 100644 --- a/vitrage/graph/driver/networkx_graph.py +++ b/vitrage/graph/driver/networkx_graph.py @@ -293,6 +293,15 @@ class NXGraph(Graph): return json.dumps(node_link_data) + def to_json(self): + return json_graph.node_link_data(self._g) + + @staticmethod + def from_json(data): + graph = NXGraph() + graph._g = nx.MultiDiGraph(json_graph.node_link_graph(data)) + return graph + def union(self, other_graph): """Union two graphs - add all vertices and edges of other graph diff --git a/vitrage/opts.py b/vitrage/opts.py index 1fef21d62..0178a47d4 100644 --- a/vitrage/opts.py +++ b/vitrage/opts.py @@ -28,7 +28,7 @@ import vitrage.machine_learning.plugins.jaccard_correlation import vitrage.notifier import vitrage.notifier.plugins.snmp import vitrage.os_clients -import vitrage.persistor +import vitrage.persistency import vitrage.rpc import vitrage.snmp_parsing import vitrage.storage @@ -48,7 +48,7 @@ def list_opts(): ('evaluator', vitrage.evaluator.OPTS), ('consistency', vitrage.entity_graph.consistency.OPTS), ('database', vitrage.storage.OPTS), - ('persistor', vitrage.persistor.OPTS), + ('persistency', vitrage.persistency.OPTS), ('entity_graph', vitrage.entity_graph.OPTS), ('service_credentials', vitrage.keystone_client.OPTS), ('machine_learning', diff --git a/vitrage/persistor/__init__.py b/vitrage/persistency/__init__.py similarity index 74% rename from vitrage/persistor/__init__.py rename to vitrage/persistency/__init__.py index 1311ed77e..71eac10b3 100644 --- a/vitrage/persistor/__init__.py +++ b/vitrage/persistency/__init__.py @@ -19,7 +19,11 @@ OPTS = [ default='vitrage_persistor', help='The topic on which event will be sent from the ' 'datasources to the persistor'), - cfg.BoolOpt('persist_events', + cfg.BoolOpt('enable_persistency', default=False, - help='Whether or not persistor is persisting the events'), + help='Periodically store the entire graph snapshot to ' + 'the database'), + cfg.IntOpt('graph_persistency_interval', + default=3600, + help='Store the graph to the database every X seconds'), ] diff --git a/vitrage/persistency/graph_persistor.py b/vitrage/persistency/graph_persistor.py new file mode 100644 index 000000000..4b97d35a0 --- /dev/null +++ b/vitrage/persistency/graph_persistor.py @@ -0,0 +1,59 @@ +# Copyright 2018 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +from oslo_log import log + +from dateutil import parser +from vitrage.common.constants import DatasourceProperties as DSProps +from vitrage.graph.driver.networkx_graph import NXGraph +from vitrage import storage +from vitrage.storage.sqlalchemy import models +from vitrage.utils import datetime +from vitrage.utils.datetime import utcnow + + +LOG = log.getLogger(__name__) + + +class GraphPersistor(object): + def __init__(self, conf): + super(GraphPersistor, self).__init__() + self.db_connection = storage.get_connection_from_config(conf) + self.last_event_timestamp = datetime.datetime.utcnow() + + def store_graph(self, graph): + try: + graph_snapshot = graph.to_json() + db_row = models.GraphSnapshot( + last_event_timestamp=self.last_event_timestamp, + graph_snapshot=graph_snapshot) + self.db_connection.graph_snapshots.create(db_row) + except Exception as e: + LOG.exception("Graph is not stored: %s", e) + + def load_graph(self, timestamp=None): + db_row = self.db_connection.graph_snapshots.query(timestamp) if \ + timestamp else self.db_connection.graph_snapshots.query(utcnow()) + return NXGraph.from_json(db_row.graph_snapshot) if db_row else None + + def delete_graph_snapshots(self, timestamp): + """Deletes all graph snapshots until timestamp""" + self.db_connection.graph_snapshots.delete(timestamp) + + def update_last_event_timestamp(self, event): + timestamp = event.get(DSProps.SAMPLE_DATE) + self.last_event_timestamp = parser.parse(timestamp) if timestamp \ + else None diff --git a/vitrage/persistor/service.py b/vitrage/persistency/service.py similarity index 97% rename from vitrage/persistor/service.py rename to vitrage/persistency/service.py index 90e21cb39..72a0df2bb 100644 --- a/vitrage/persistor/service.py +++ b/vitrage/persistency/service.py @@ -35,7 +35,7 @@ class PersistorService(os_service.Service): self.db_connection = db_connection transport = messaging.get_transport(conf) target = \ - oslo_m.Target(topic=conf.persistor.persistor_topic) + oslo_m.Target(topic=conf.persistency.persistor_topic) self.listener = messaging.get_notification_listener( transport, [target], [VitragePersistorEndpoint(self.db_connection)]) diff --git a/vitrage/storage/base.py b/vitrage/storage/base.py index abdaa2493..fa372343e 100644 --- a/vitrage/storage/base.py +++ b/vitrage/storage/base.py @@ -35,6 +35,10 @@ class Connection(object): def templates(self): return None + @property + def graph_snapshots(self): + return None + @abc.abstractmethod def upgrade(self, nocreate=False): raise NotImplementedError('upgrade not implemented') @@ -165,8 +169,35 @@ class EventsConnection(object): def delete(self, event_id=None, collector_timestamp=None, - payload=None, gt_collector_timestamp=None, lt_collector_timestamp=None): """Delete all events that match the filters.""" raise NotImplementedError('delete events not implemented') + + +@six.add_metaclass(abc.ABCMeta) +class GraphSnapshotsConnection(object): + def create(self, graph_snapshot): + """Create a new graph snapshot. + + :type graph_snapshot: vitrage.storage.sqlalchemy.models.GraphSnapshot + """ + raise NotImplementedError('create graph snapshot not implemented') + + def update(self, graph_snapshot): + """Update a graph snapshot. + + :type graph_snapshot: vitrage.storage.sqlalchemy.models.GraphSnapshot + """ + raise NotImplementedError('update graph snapshot not implemented') + + def query(self, timestamp=None): + """Yields latest graph snapshot taken until timestamp. + + :rtype: vitrage.storage.sqlalchemy.models.GraphSnapshot + """ + raise NotImplementedError('query graph snapshot not implemented') + + def delete(self, timestamp=None): + """Delete all graph snapshots taken until timestamp.""" + raise NotImplementedError('delete graph snapshots not implemented') diff --git a/vitrage/storage/impl_sqlalchemy.py b/vitrage/storage/impl_sqlalchemy.py index 1eb772d2d..ae0fbeb06 100644 --- a/vitrage/storage/impl_sqlalchemy.py +++ b/vitrage/storage/impl_sqlalchemy.py @@ -19,6 +19,7 @@ from oslo_db.sqlalchemy import session as db_session from oslo_log import log from sqlalchemy.engine import url as sqlalchemy_url +from vitrage.common.exception import VitrageInputError from vitrage import storage from vitrage.storage import base from vitrage.storage.sqlalchemy import models @@ -42,6 +43,7 @@ class Connection(base.Connection): self._active_actions = ActiveActionsConnection(self._engine_facade) self._events = EventsConnection(self._engine_facade) self._templates = TemplatesConnection(self._engine_facade) + self._graph_snapshots = GraphSnapshotsConnection(self._engine_facade) @property def active_actions(self): @@ -55,6 +57,10 @@ class Connection(base.Connection): def templates(self): return self._templates + @property + def graph_snapshots(self): + return self._graph_snapshots + @staticmethod def _dress_url(url): # If no explicit driver has been set, we default to pymysql @@ -206,6 +212,19 @@ class EventsConnection(base.EventsConnection, BaseTableConn): payload=None, gt_collector_timestamp=None, lt_collector_timestamp=None): + """Yields a lists of events that match filters. + + :raises: vitrage.common.exception.VitrageInputError. + :rtype: list of vitrage.storage.sqlalchemy.models.Event + """ + + if (event_id or collector_timestamp or payload) and \ + (gt_collector_timestamp or lt_collector_timestamp): + msg = "Calling function with both specific event and range of " \ + "events parameters at the same time " + LOG.debug(msg) + raise VitrageInputError(msg) + query = self.query_filter( models.Event, event_id=event_id, @@ -233,17 +252,56 @@ class EventsConnection(base.EventsConnection, BaseTableConn): def delete(self, event_id=None, collector_timestamp=None, - payload=None, gt_collector_timestamp=None, lt_collector_timestamp=None): + """Delete all events that match the filters. + + :raises: vitrage.common.exception.VitrageInputError. + """ + if (event_id or collector_timestamp) and \ + (gt_collector_timestamp or lt_collector_timestamp): + msg = "Calling function with both specific event and range of " \ + "events parameters at the same time " + LOG.debug(msg) + raise VitrageInputError(msg) + query = self.query_filter( models.Event, event_id=event_id, - collector_timestamp=collector_timestamp, - payload=payload) + collector_timestamp=collector_timestamp) query = self._update_query_gt_lt(gt_collector_timestamp, lt_collector_timestamp, query) query.delete() + + +class GraphSnapshotsConnection(base.GraphSnapshotsConnection, BaseTableConn): + def __init__(self, engine_facade): + super(GraphSnapshotsConnection, self).__init__(engine_facade) + + def create(self, graph_snapshot): + session = self._engine_facade.get_session() + with session.begin(): + session.add(graph_snapshot) + + def update(self, graph_snapshot): + session = self._engine_facade.get_session() + with session.begin(): + session.merge(graph_snapshot) + + def query(self, timestamp=None): + query = self.query_filter(models.GraphSnapshot) + query = query.filter(models.GraphSnapshot.last_event_timestamp <= + timestamp) + return query.order_by( + models.GraphSnapshot.last_event_timestamp.desc()).first() + + def delete(self, timestamp=None): + """Delete all graph snapshots taken until timestamp.""" + query = self.query_filter(models.GraphSnapshot) + + query = query.filter(models.GraphSnapshot.last_event_timestamp <= + timestamp) + query.delete() diff --git a/vitrage/storage/sqlalchemy/models.py b/vitrage/storage/sqlalchemy/models.py index 36f72342f..581712af6 100644 --- a/vitrage/storage/sqlalchemy/models.py +++ b/vitrage/storage/sqlalchemy/models.py @@ -121,6 +121,23 @@ class ActiveAction(Base, models.TimestampMixin): ) +class GraphSnapshot(Base): + __tablename__ = 'graph_snapshots' + + last_event_timestamp = Column(DateTime, primary_key=True, nullable=False) + graph_snapshot = Column(JSONEncodedDict(), nullable=False) + + def __repr__(self): + return \ + "" %\ + ( + self.last_event_timestamp, + self.graph_snapshot + ) + + class Template(Base, models.TimestampMixin): __tablename__ = 'templates' diff --git a/vitrage/tests/base.py b/vitrage/tests/base.py index 5864d259f..25d555023 100644 --- a/vitrage/tests/base.py +++ b/vitrage/tests/base.py @@ -65,6 +65,29 @@ class BaseTest(base.BaseTestCase): except (TypeError, AttributeError): self.fail("%s doesn't have length" % type(obj)) + def assert_graph_equal(self, g1, g2): + """Checks that two graphs are equals. + + This relies on assert_dict_equal when comparing the nodes and the + edges of each graph. + """ + g1_nodes = g1._g.node + g1_edges = g1._g.edge + g2_nodes = g2._g.node + g2_edges = g2._g.edge + self.assertEqual(g1.num_vertices(), g2.num_vertices(), + "Two graphs have different amount of nodes") + self.assertEqual(g1.num_edges(), g2.num_edges(), + "Two graphs have different amount of edges") + for n_id in g1_nodes: + self.assert_dict_equal(g1_nodes.get(n_id), + g2_nodes.get(n_id), + "Nodes of each graph are not equal") + for e_source_id in g1_edges: + self.assert_dict_equal(g1_edges.get(e_source_id), + g2_edges.get(e_source_id), + "Edges of each graph are not equal") + @staticmethod def path_get(project_file=None): root = os.path.abspath(os.path.join(os.path.dirname(__file__), diff --git a/vitrage/tests/functional/entity_graph/graph_persistor/__init__.py b/vitrage/tests/functional/entity_graph/graph_persistor/__init__.py new file mode 100644 index 000000000..5c2b8b158 --- /dev/null +++ b/vitrage/tests/functional/entity_graph/graph_persistor/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2018 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__author__ = 'stack' diff --git a/vitrage/tests/functional/entity_graph/graph_persistor/test_graph_persistor.py b/vitrage/tests/functional/entity_graph/graph_persistor/test_graph_persistor.py new file mode 100644 index 000000000..6e6c71a18 --- /dev/null +++ b/vitrage/tests/functional/entity_graph/graph_persistor/test_graph_persistor.py @@ -0,0 +1,94 @@ +# Copyright 2018 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import time + +from oslo_config import cfg +from oslo_db.options import database_opts + +from vitrage.persistency.graph_persistor import GraphPersistor +from vitrage import storage +from vitrage.storage.sqlalchemy import models +from vitrage.tests.functional.base import TestFunctionalBase +from vitrage.tests.mocks.graph_generator import GraphGenerator +from vitrage.utils.datetime import utcnow + + +class TestGraphPersistor(TestFunctionalBase): + + # noinspection PyAttributeOutsideInit,PyPep8Naming + @classmethod + def setUpClass(cls): + super(TestGraphPersistor, cls).setUpClass() + cls.conf = cfg.ConfigOpts() + cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph') + cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources') + cls.conf.register_opts(database_opts, group='database') + cls.conf.set_override('connection', 'sqlite:///:test.db:', + group='database') + cls._db = storage.get_connection_from_config(cls.conf) + engine = cls._db._engine_facade.get_engine() + models.Base.metadata.create_all(engine) + cls.load_datasources(cls.conf) + cls.graph_persistor = GraphPersistor(cls.conf) + + def test_persist_graph(self): + g = GraphGenerator().create_graph() + current_time = utcnow() + self.graph_persistor.last_event_timestamp = current_time + self.graph_persistor.store_graph(g) + graph_snapshot = self.graph_persistor.load_graph(current_time) + self.assert_graph_equal(g, graph_snapshot) + self.graph_persistor.delete_graph_snapshots(utcnow()) + + def test_persist_two_graphs(self): + g1 = GraphGenerator().create_graph() + current_time1 = utcnow() + self.graph_persistor.last_event_timestamp = current_time1 + self.graph_persistor.store_graph(g1) + graph_snapshot1 = self.graph_persistor.load_graph(current_time1) + + g2 = GraphGenerator(5).create_graph() + current_time2 = utcnow() + self.graph_persistor.last_event_timestamp = current_time2 + self.graph_persistor.store_graph(g2) + graph_snapshot2 = self.graph_persistor.load_graph(current_time2) + + self.assert_graph_equal(g1, graph_snapshot1) + self.assert_graph_equal(g2, graph_snapshot2) + self.graph_persistor.delete_graph_snapshots(utcnow()) + + def test_load_last_graph_snapshot_until_timestamp(self): + g1 = GraphGenerator().create_graph() + self.graph_persistor.last_event_timestamp = utcnow() + self.graph_persistor.store_graph(g1) + + time.sleep(1) + time_in_between = utcnow() + time.sleep(1) + + g2 = GraphGenerator(5).create_graph() + self.graph_persistor.last_event_timestamp = utcnow() + self.graph_persistor.store_graph(g2) + + graph_snapshot = self.graph_persistor.load_graph(time_in_between) + self.assert_graph_equal(g1, graph_snapshot) + self.graph_persistor.delete_graph_snapshots(utcnow()) + + def test_delete_graph_snapshots(self): + g = GraphGenerator().create_graph() + self.graph_persistor.last_event_timestamp = utcnow() + self.graph_persistor.store_graph(g) + self.graph_persistor.delete_graph_snapshots(utcnow()) + graph_snapshot = self.graph_persistor.load_graph(utcnow()) + self.assertIsNone(graph_snapshot) diff --git a/vitrage/tests/mocks/graph_generator.py b/vitrage/tests/mocks/graph_generator.py new file mode 100644 index 000000000..363a85e59 --- /dev/null +++ b/vitrage/tests/mocks/graph_generator.py @@ -0,0 +1,169 @@ +# Copyright 2018 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import itertools + +from vitrage.common.constants import EdgeProperties +from vitrage.graph import Direction +from vitrage.graph.driver.networkx_graph import NXGraph +from vitrage.graph import Edge +from vitrage.graph import Vertex +from vitrage.tests.mocks import utils + +RESOURCES_PATH = utils.get_resources_dir() + '/mock_configurations' + + +class GraphGenerator(object): + def __init__(self, + num_of_networks=2, + num_of_zones_per_cluster=2, + num_of_hosts_per_zone=2, + num_of_zabbix_alarms_per_host=2, + num_of_instances_per_host=2, + num_of_ports_per_instance=2, + num_of_volumes_per_instance=2, + num_of_vitrage_alarms_per_instance=2, + num_of_tripleo_controllers=2, + num_of_zabbix_alarms_per_controller=2): + self.id_counter = 0 + self._num_of_networks = num_of_networks + self._num_of_zones_per_cluster = num_of_zones_per_cluster + self._num_of_hosts_per_zone = num_of_hosts_per_zone + self._num_of_zabbix_alarms_per_host = num_of_zabbix_alarms_per_host + self._num_of_instances_per_host = num_of_instances_per_host + self._num_of_ports_per_instance = num_of_ports_per_instance + self._num_of_volumes_per_instance = num_of_volumes_per_instance + self._num_of_vitrage_alarms_per_instance = \ + num_of_vitrage_alarms_per_instance + self._num_of_tripleo_controllers = num_of_tripleo_controllers + self._num_of_zabbix_alarms_per_controller = \ + num_of_zabbix_alarms_per_controller + + def create_graph(self): + graph = NXGraph() + v1 = self._file_to_vertex('openstack-cluster.json') + graph.add_vertex(v1) + + networks = self._create_n_vertices(graph, + self._num_of_networks, + 'neutron.network.json') + zones = self._create_n_neighbors(graph, + self._num_of_zones_per_cluster, + [v1], + 'nova.zone.json', + 'contains.json') + hosts = self._create_n_neighbors(graph, + self._num_of_hosts_per_zone, + zones, + 'nova.host.json', + 'contains.json') + self._create_n_neighbors(graph, + self._num_of_zabbix_alarms_per_host, + hosts, + 'zabbix.json', + 'on.json', + Direction.IN) + instances = self._create_n_neighbors(graph, + self._num_of_instances_per_host, + hosts, + 'nova.instance.json', + 'contains.json') + ports = self._create_n_neighbors(graph, + self._num_of_ports_per_instance, + instances, + 'neutron.port.json', + 'attached.json', + direction=Direction.IN) + + self._round_robin_edges(graph, networks, ports, 'contains.json') + + self._create_n_neighbors(graph, + self._num_of_volumes_per_instance, + instances, + 'cinder.volume.json', + 'attached.json', + Direction.IN) + self._create_n_neighbors(graph, + self._num_of_vitrage_alarms_per_instance, + instances, + 'vitrage.alarm.json', + 'on.json', + Direction.IN) + + # Also create non connected components: + tripleo_controller = \ + self._create_n_vertices(graph, + self._num_of_tripleo_controllers, + 'tripleo.controller.json') + self._create_n_neighbors(graph, + self._num_of_zabbix_alarms_per_controller, + tripleo_controller, + 'zabbix.json', + 'on.json', + Direction.IN) + return graph + + def _create_n_vertices(self, g, n, props_file): + created_vertices = [] + for i in range(n): + v = self._file_to_vertex(props_file) + created_vertices.append(v) + g.add_vertex(v) + return created_vertices + + def _create_n_neighbors(self, g, n, source_v_list, + neighbor_props_file, neighbor_edge_props_file, + direction=Direction.OUT): + created_vertices = [] + for source_v in source_v_list: + for i in range(n): + v = self._file_to_vertex(neighbor_props_file) + created_vertices.append(v) + g.add_vertex(v) + if direction == Direction.OUT: + g.add_edge(self._file_to_edge(neighbor_edge_props_file, + source_v.vertex_id, + v.vertex_id)) + else: + g.add_edge( + self._file_to_edge(neighbor_edge_props_file, + v.vertex_id, + source_v.vertex_id)) + return created_vertices + + def _round_robin_edges(self, + graph, + source_vertices, + target_vertices, + edge_props_file): + round_robin_source_vertices = itertools.cycle(source_vertices) + for v in target_vertices: + source_v = next(round_robin_source_vertices) + graph.add_edge(self._file_to_edge(edge_props_file, + source_v.vertex_id, + v.vertex_id)) + + def _file_to_vertex(self, relative_path): + full_path = RESOURCES_PATH + "/vertices/" + props = utils.load_specs(relative_path, full_path) + v = Vertex(str(self.id_counter), props) + self.id_counter += 1 + return v + + @staticmethod + def _file_to_edge(relative_path, source_id, target_id): + full_path = RESOURCES_PATH + "/edges/" + props = utils.load_specs(relative_path, full_path) + return Edge(source_id, target_id, + props[EdgeProperties.RELATIONSHIP_TYPE], + props) diff --git a/vitrage/tests/resources/mock_configurations/driver/driver_stack_snapshot_dynamic.json b/vitrage/tests/resources/mock_configurations/driver/driver_stack_snapshot_dynamic.json index cc5bc9e2a..42a6789b7 100644 --- a/vitrage/tests/resources/mock_configurations/driver/driver_stack_snapshot_dynamic.json +++ b/vitrage/tests/resources/mock_configurations/driver/driver_stack_snapshot_dynamic.json @@ -7,7 +7,7 @@ "vitrage_datasource_action": "init_snapshot", "id": "f22416fb-b33c-4e24-822e-0174421f5ece", "stack_status": "CREATE_COMPLETE", - "vitrage_sample_date": "2016-08-2515:12:28.281460+00:00", + "vitrage_sample_date": "2017-12-13 11:24:02.598358+00:00", "vitrage_entity_type": "heat.stack", "resources": [{ "resource_name": "cinder_volume_1", diff --git a/vitrage/tests/resources/mock_configurations/edges/attached.json b/vitrage/tests/resources/mock_configurations/edges/attached.json new file mode 100644 index 000000000..759520cc7 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/edges/attached.json @@ -0,0 +1,5 @@ +{ + "vitrage_is_deleted": false, + "update_timestamp": "2017-12-24T10:32:48Z", + "relationship_type": "attached" +} diff --git a/vitrage/tests/resources/mock_configurations/edges/contains.json b/vitrage/tests/resources/mock_configurations/edges/contains.json new file mode 100644 index 000000000..d379f772d --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/edges/contains.json @@ -0,0 +1,5 @@ +{ + "vitrage_is_deleted": false, + "update_timestamp": "2017-12-24T10:32:48Z", + "relationship_type": "contains" +} diff --git a/vitrage/tests/resources/mock_configurations/edges/on.json b/vitrage/tests/resources/mock_configurations/edges/on.json new file mode 100644 index 000000000..9d8063ef6 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/edges/on.json @@ -0,0 +1,5 @@ +{ + "vitrage_is_deleted": false, + "update_timestamp": "2017-12-24T10:32:48Z", + "relationship_type": "on" +} diff --git a/vitrage/tests/resources/mock_configurations/vertices/cinder.volume.json b/vitrage/tests/resources/mock_configurations/vertices/cinder.volume.json new file mode 100644 index 000000000..c18821102 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/cinder.volume.json @@ -0,0 +1,19 @@ +{ + "vitrage_id": "12b49b03-e754-4728-8214-4ad20ae5b187", + "vitrage_is_deleted": false, + "update_timestamp": "2017-11-09T11:18:44.000000", + "size": 1, + "vitrage_category": "RESOURCE", + "volume_type": "lvmdriver-1", + "vitrage_operational_state": "OK", + "state": "in-use", + "vitrage_type": "cinder.volume", + "vitrage_sample_timestamp": "2017-12-25 09:43:49.905125+00:00", + "vitrage_aggregated_state": "IN-USE", + "vitrage_is_placeholder": false, + "project_id": "7ff7bcc9c23d48b9afe7de8029981c22", + "is_real_vitrage_id": true, + "attachments": [ + "f96f3054-41fc-4110-a182-336fbb2168fc" + ] +} \ No newline at end of file diff --git a/vitrage/tests/resources/mock_configurations/vertices/neutron.network.json b/vitrage/tests/resources/mock_configurations/vertices/neutron.network.json new file mode 100644 index 000000000..f70edeb35 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/neutron.network.json @@ -0,0 +1,15 @@ +{ + "vitrage_id": "b951c60e-8815-45b7-900a-052816cc2515", + "name": "private", + "update_timestamp": "2017-11-20T13:49:13Z", + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "OK", + "state": "ACTIVE", + "vitrage_type": "neutron.network", + "vitrage_sample_timestamp": "2017-12-25 06:30:24.928811+00:00", + "vitrage_aggregated_state": "ACTIVE", + "vitrage_is_placeholder": false, + "project_id": "c0879e8fe5084cd89af29514ec4fddfe", + "is_real_vitrage_id": true, + "vitrage_is_deleted": false +} diff --git a/vitrage/tests/resources/mock_configurations/vertices/neutron.port.json b/vitrage/tests/resources/mock_configurations/vertices/neutron.port.json new file mode 100644 index 000000000..f7df72a69 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/neutron.port.json @@ -0,0 +1,18 @@ +{ + "vitrage_id": "27432eb2-3e09-4e50-b3bf-818075b376ea", + "vitrage_is_deleted": false, + "update_timestamp": "2017-11-20T13:49:43Z", + "ip_addresses": [ + "10.0.0.1" + ], + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "OK", + "state": "ACTIVE", + "vitrage_type": "neutron.port", + "vitrage_sample_timestamp": "2017-12-25 06:30:25.387277+00:00", + "host_id": "compute-0-0", + "vitrage_aggregated_state": "ACTIVE", + "vitrage_is_placeholder": false, + "project_id": "c0879e8fe5084cd89af29514ec4fddfe", + "is_real_vitrage_id": true +} \ No newline at end of file diff --git a/vitrage/tests/resources/mock_configurations/vertices/nova.host.json b/vitrage/tests/resources/mock_configurations/vertices/nova.host.json new file mode 100644 index 000000000..f86b9b891 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/nova.host.json @@ -0,0 +1,11 @@ +{ + "vitrage_id": "e2e5054e-f3bd-49e7-b584-04fb1fbc0e3f", + "vitrage_is_deleted": false, + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "N/A", + "vitrage_type": "nova.host", + "vitrage_sample_timestamp": "2017-12-24 10:32:41.389676+00:00", + "vitrage_aggregated_state": null, + "vitrage_is_placeholder": false, + "is_real_vitrage_id": true +} diff --git a/vitrage/tests/resources/mock_configurations/vertices/nova.instance.json b/vitrage/tests/resources/mock_configurations/vertices/nova.instance.json new file mode 100644 index 000000000..2da5d2bfb --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/nova.instance.json @@ -0,0 +1,17 @@ +{ + "vitrage_id": "29f18c8b-1fce-4abb-8e96-4b478e124c59", + "vitrage_state": "SUBOPTIMAL", + "vitrage_is_deleted": false, + "update_timestamp": "2017-12-25 09:43:49.711469+00:00", + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "SUBOPTIMAL", + "state": "ACTIVE", + "vitrage_type": "nova.instance", + "vitrage_sample_timestamp": "2017-12-25 09:43:49.711469+00:00", + "host_id": "compute-0-0", + "vitrage_aggregated_state": "SUBOPTIMAL", + "vitrage_is_placeholder": false, + "project_id": "7ff7bcc9c23d48b9afe7de8029981c22", + "is_real_vitrage_id": true, + "name": "App_2-server_1-esi2oaogigfp" +} \ No newline at end of file diff --git a/vitrage/tests/resources/mock_configurations/vertices/nova.zone.json b/vitrage/tests/resources/mock_configurations/vertices/nova.zone.json new file mode 100644 index 000000000..37fe892e6 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/nova.zone.json @@ -0,0 +1,14 @@ +{ + "vitrage_id": "1572855d-1551-4507-b60c-3a16ddc012bf", + "name": "nova", + "update_timestamp": "2017-12-24 10:32:41.941967+00:00", + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "OK", + "state": "available", + "vitrage_type": "nova.zone", + "vitrage_sample_timestamp": "2017-12-24 10:32:41.941967+00:00", + "vitrage_aggregated_state": "AVAILABLE", + "vitrage_is_placeholder": false, + "is_real_vitrage_id": true, + "vitrage_is_deleted": false +} diff --git a/vitrage/tests/resources/mock_configurations/vertices/openstack-cluster.json b/vitrage/tests/resources/mock_configurations/vertices/openstack-cluster.json new file mode 100644 index 000000000..e4e8d94b6 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/openstack-cluster.json @@ -0,0 +1,13 @@ +{ + "vitrage_id": "4bfbde3a-bb30-4c0a-b1e4-83ebfb2ec2ff", + "vitrage_is_deleted": false, + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "OK", + "state": "available", + "vitrage_type": "openstack.cluster", + "vitrage_sample_timestamp": "2017-12-24 10:32:41.941967+00:00", + "vitrage_aggregated_state": "AVAILABLE", + "vitrage_is_placeholder": false, + "is_real_vitrage_id": true, + "name": "openstack.cluster" +} diff --git a/vitrage/tests/resources/mock_configurations/vertices/tripleo.controller.json b/vitrage/tests/resources/mock_configurations/vertices/tripleo.controller.json new file mode 100644 index 000000000..6a8bbd1a6 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/tripleo.controller.json @@ -0,0 +1,13 @@ +{ + "vitrage_id": "2396b5bb-b00b-4d6d-b842-6debb3ba8091", + "name": "overcloud-controller-0.localdomain", + "update_timestamp": "2017-12-25 09:33:05.073194+00:00", + "vitrage_category": "RESOURCE", + "vitrage_operational_state": "OK", + "state": "ACTIVE", + "vitrage_type": "tripleo.controller", + "vitrage_sample_timestamp": "2017-12-25 09:33:05.073194+00:00", + "vitrage_aggregated_state": "ACTIVE", + "vitrage_is_placeholder": false, + "vitrage_is_deleted": false +} \ No newline at end of file diff --git a/vitrage/tests/resources/mock_configurations/vertices/vitrage.alarm.json b/vitrage/tests/resources/mock_configurations/vertices/vitrage.alarm.json new file mode 100644 index 000000000..59bb7c971 --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/vitrage.alarm.json @@ -0,0 +1,18 @@ +{ + "vitrage_id": "98fab0f9-72b7-4362-a16c-1b19d72505c4", + "vitrage_is_deleted": false, + "update_timestamp": "2017-12-24T10:32:49Z", + "resource_id": "2507553d-1738-4711-938f-19a47181bfc1", + "severity": "critical", + "vitrage_category": "ALARM", + "state": "Active", + "vitrage_type": "vitrage", + "vitrage_sample_timestamp": "2017-12-24 10:32:49.063574+00:00", + "vitrage_operational_severity": "CRITICAL", + "vitrage_is_placeholder": false, + "vitrage_aggregated_severity": "CRITICAL", + "vitrage_resource_id": "2507553d-1738-4711-938f-19a47181bfc1", + "vitrage_resource_type": "nova.instance", + "is_real_vitrage_id": true, + "name": "VM network problem 3" +} diff --git a/vitrage/tests/resources/mock_configurations/vertices/zabbix.json b/vitrage/tests/resources/mock_configurations/vertices/zabbix.json new file mode 100644 index 000000000..19e6b7d8b --- /dev/null +++ b/vitrage/tests/resources/mock_configurations/vertices/zabbix.json @@ -0,0 +1,18 @@ +{ + "rawtext": "Component etcd-1 is not in Healthy state", + "vitrage_id": "a508142a-b0c6-4880-8b05-38d252b0d840", + "name": "Component etcd-1 is not in Healthy state", + "update_timestamp": "2017-12-24T10:32:43Z", + "resource_id": "k8s", + "vitrage_category": "ALARM", + "vitrage_is_deleted": false, + "state": "Active", + "vitrage_type": "zabbix", + "vitrage_sample_timestamp": "2017-12-25 09:33:02.733159+00:00", + "vitrage_operational_severity": "WARNING", + "vitrage_is_placeholder": false, + "vitrage_aggregated_severity": "WARNING", + "vitrage_resource_id": "932d6f9a-dfc4-4a32-9e7a-1fb16a68d3af", + "vitrage_resource_type": "kubernetes_cluster", + "severity": "WARNING" +} \ No newline at end of file diff --git a/vitrage_tempest_tests/tests/api/rca/test_rca.py b/vitrage_tempest_tests/tests/api/rca/test_rca.py index c4435e0bb..9ce01020d 100644 --- a/vitrage_tempest_tests/tests/api/rca/test_rca.py +++ b/vitrage_tempest_tests/tests/api/rca/test_rca.py @@ -39,7 +39,7 @@ class TestRca(BaseRcaTest): """compare_cli_and_api test There test validate correctness of rca of created - aodh event alarms, and compare them with cli rca + aodh event alarms, and equals them with cli rca """ try: instances = nova_utils.create_instances(num_instances=1, @@ -97,7 +97,7 @@ class TestRca(BaseRcaTest): """validate_deduce_alarms test There tests validates correctness of deduce alarms - (created by special template file), and compare there + (created by special template file), and equals there resource_id with created instances id """ try: diff --git a/vitrage_tempest_tests/tests/api/templates/test_template.py b/vitrage_tempest_tests/tests/api/templates/test_template.py index 425e6bbff..7118d21a6 100644 --- a/vitrage_tempest_tests/tests/api/templates/test_template.py +++ b/vitrage_tempest_tests/tests/api/templates/test_template.py @@ -33,7 +33,7 @@ class TestValidate(BaseTemplateTest): """template_list test There test validate correctness of template list, - compare templates files existence with default folder + equals templates files existence with default folder and between cli via api ... """ api_template_list = self.vitrage_client.template.list() @@ -46,7 +46,7 @@ class TestValidate(BaseTemplateTest): """template_validate test There test validate correctness of template validation, - compare templates files validation between cli via api + equals templates files validation between cli via api """ path = self.DEFAULT_PATH api_template_validation = \