From 0b0980f5f2940dfcddd198411a17ec23b51774ec Mon Sep 17 00:00:00 2001 From: Idan Hefetz Date: Tue, 5 Jun 2018 11:33:27 +0000 Subject: [PATCH] Add a new tempest job with a mock datasource In this tempest job, the mock datasource is used to create an entity graph with ~20,000 vertices. This job will contain tests for init procedure, processes, restarts, multiple api calls. Other data sources are disabled. Heat, Telemetry, Mistral are not installed. This commit includes the first two tests, both using a template where each action causes another action (three levels). one test restarts vitrage-graph so the init procedure evaluates this template. the other test does not restart. result should be the same. Change-Id: I9c4c329983bbe5930de5484fa6932b318746ed5c --- devstack/post_test_hook.sh | 2 + .../tests/common/vitrage_utils.py | 18 ++- .../tests/resources/__init__.py | 0 .../resources/mock_datasource/__init__.py | 0 .../test_3rd_degree_scenarios.py | 117 ++++++++++++++++++ .../mock_datasource_3rd_degree_scenarios.yaml | 88 +++++++++++++ 6 files changed, 222 insertions(+), 3 deletions(-) create mode 100644 vitrage_tempest_plugin/tests/resources/__init__.py create mode 100644 vitrage_tempest_plugin/tests/resources/mock_datasource/__init__.py create mode 100644 vitrage_tempest_plugin/tests/resources/mock_datasource/test_3rd_degree_scenarios.py create mode 100644 vitrage_tempest_plugin/tests/resources/templates/api/mock_datasource_3rd_degree_scenarios.yaml diff --git a/devstack/post_test_hook.sh b/devstack/post_test_hook.sh index 3879881..b9cc986 100644 --- a/devstack/post_test_hook.sh +++ b/devstack/post_test_hook.sh @@ -20,6 +20,8 @@ if [ "$1" = "api" ]; then TESTS="topology|test_rca|test_alarms|test_resources|test_template|test_webhook" elif [ "$1" = "datasources" ]; then TESTS="datasources|test_events|notifiers|e2e|database" +elif [ "$1" = "mock" ]; then + TESTS="mock_datasource" else TESTS="topology" fi diff --git a/vitrage_tempest_plugin/tests/common/vitrage_utils.py b/vitrage_tempest_plugin/tests/common/vitrage_utils.py index fb1d186..49b3b92 100644 --- a/vitrage_tempest_plugin/tests/common/vitrage_utils.py +++ b/vitrage_tempest_plugin/tests/common/vitrage_utils.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. from datetime import datetime +import os + from oslo_log import log as logging from vitrage.common.constants import TemplateStatus @@ -67,7 +69,7 @@ def add_template(filename='', t = TempestClients.vitrage().template.add(full_path, template_type) if t and t[0]: wait_for_status( - 10, + 100, get_first_template, uuid=t[0]['uuid'], status=TemplateStatus.ACTIVE) return t[0] @@ -79,9 +81,19 @@ def get_first_template(**kwargs): return g_utils.first_match(templates, **kwargs) -def delete_template(uuid): +def delete_template(uuid=None, **kwargs): + if not uuid: + template = get_first_template(**kwargs) + if template: + uuid = template['uuid'] + else: + return TempestClients.vitrage().template.delete(uuid) wait_for_status( - 10, + 100, lambda _id: True if not get_first_template(uuid=_id) else False, _id=uuid) + + +def restart_graph(): + os.system("sudo service devstack@vitrage-graph restart") diff --git a/vitrage_tempest_plugin/tests/resources/__init__.py b/vitrage_tempest_plugin/tests/resources/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vitrage_tempest_plugin/tests/resources/mock_datasource/__init__.py b/vitrage_tempest_plugin/tests/resources/mock_datasource/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vitrage_tempest_plugin/tests/resources/mock_datasource/test_3rd_degree_scenarios.py b/vitrage_tempest_plugin/tests/resources/mock_datasource/test_3rd_degree_scenarios.py new file mode 100644 index 0000000..1d4360d --- /dev/null +++ b/vitrage_tempest_plugin/tests/resources/mock_datasource/test_3rd_degree_scenarios.py @@ -0,0 +1,117 @@ +# Copyright 2018 - Nokia +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from concurrent import futures +import time + +from oslo_log import log as logging + +from vitrage_tempest_plugin.tests.common.constants import VertexProperties +from vitrage_tempest_plugin.tests.common import general_utils as g_utils +from vitrage_tempest_plugin.tests.common.tempest_clients import TempestClients +from vitrage_tempest_plugin.tests.common import vitrage_utils as v_utils +from vitrage_tempest_plugin.tests.e2e.test_actions_base import TestActionsBase +from vitrage_tempest_plugin.tests import utils + +LOG = logging.getLogger(__name__) + +DEDUCED_1 = 'mock_datasource.3rd_degree_scenarios.deduced.alarm1' +DEDUCED_2 = 'mock_datasource.3rd_degree_scenarios.deduced.alarm2' +SLEEP = 80 + + +class TestLongProcessing(TestActionsBase): + + @classmethod + def setUpClass(cls): + super(TestLongProcessing, cls).setUpClass() + v_utils.delete_template( + name="mock_datasource_3rd_degree_scenarios.yaml") + time.sleep(SLEEP) + + @utils.tempest_logger + def test_3rd_degree_scenarios(self): + try: + v_utils.add_template("mock_datasource_3rd_degree_scenarios.yaml") + time.sleep(SLEEP) + + self._check_template_instance_3rd_degree_scenarios() + except Exception as e: + self._handle_exception(e) + raise + finally: + v_utils.delete_template( + name="mock_datasource_3rd_degree_scenarios.yaml") + time.sleep(SLEEP) + + @utils.tempest_logger + def test_3rd_degree_scenarios_init_procedure(self): + + try: + v_utils.add_template("mock_datasource_3rd_degree_scenarios.yaml") + time.sleep(SLEEP) + + v_utils.restart_graph() + time.sleep(SLEEP) + + self._check_template_instance_3rd_degree_scenarios() + except Exception as e: + self._handle_exception(e) + raise + finally: + v_utils.delete_template( + name="mock_datasource_3rd_degree_scenarios.yaml") + time.sleep(SLEEP) + + def _check_template_instance_3rd_degree_scenarios(self): + + try: + alarm_count = TempestClients.vitrage().alarm.count( + all_tenants=True) + self.assertEqual( + self.conf.mock_graph_datasource.instances_per_host, + alarm_count['SEVERE'], + 'Each instance should have one SEVERE deduced alarm') + self.assertEqual( + self.conf.mock_graph_datasource.instances_per_host, + alarm_count['CRITICAL'], + 'Each instance should have one CRITICAL deduced alarm') + + expected_rca = [{VertexProperties.VITRAGE_TYPE: 'zabbix'}] * self.\ + conf.mock_graph_datasource.zabbix_alarms_per_host + expected_rca.extend([{'name': DEDUCED_1}, {'name': DEDUCED_2}]) + + def check_rca(alarm): + rca = TempestClients.vitrage().rca.get(alarm['vitrage_id']) + try: + self._check_rca(rca, expected_rca, alarm) + return True + except Exception as e: + LOG.exception('check_rca failed', e) + return False + + # 10 threads calling rca api + alarms = TempestClients.vitrage().alarm.list(all_tenants=True, + vitrage_id='all') + deduced_alarms = g_utils.all_matches( + alarms, vitrage_type='vitrage', name=DEDUCED_2) + workers = futures.ThreadPoolExecutor(max_workers=10) + workers_result = [r for r in workers.map(check_rca, + deduced_alarms)] + self.assertTrue(all(workers_result)) + + except Exception as e: + v_utils.delete_template( + name="mock_datasource_3rd_degree_scenarios.yaml") + self._handle_exception(e) + raise diff --git a/vitrage_tempest_plugin/tests/resources/templates/api/mock_datasource_3rd_degree_scenarios.yaml b/vitrage_tempest_plugin/tests/resources/templates/api/mock_datasource_3rd_degree_scenarios.yaml new file mode 100644 index 0000000..3a98a11 --- /dev/null +++ b/vitrage_tempest_plugin/tests/resources/templates/api/mock_datasource_3rd_degree_scenarios.yaml @@ -0,0 +1,88 @@ +metadata: + name: mock_datasource_3rd_degree_scenarios.yaml + description: Raise a deduced alarm per instance on host, then per each deduced raise another deduced, then add causal relationship between the two + version: 2 + type: standard +definitions: + entities: + - entity: + category: ALARM + type: vitrage + name: mock_datasource.3rd_degree_scenarios.deduced.alarm1 + template_id: deduced_alarm + - entity: + category: ALARM + type: vitrage + name: mock_datasource.3rd_degree_scenarios.deduced.alarm2 + template_id: second_iteration_deduced_alarm + - entity: + category: ALARM + type: zabbix + template_id: zabbix_alarm + - entity: + category: RESOURCE + type: nova.host + name: nova.host-0-nova.zone-0-openstack.cluster-0 + template_id: host + - entity: + category: RESOURCE + type: nova.instance + template_id: instance + relationships: + - relationship: + source: deduced_alarm + relationship_type: on + target: instance + template_id : deduced_alarm_on_instance + - relationship: + source: second_iteration_deduced_alarm + relationship_type: on + target: instance + template_id : second_iteration_deduced_alarm_on_instance + - relationship: + source: host + target: instance + relationship_type: contains + template_id: host_contains_instance + - relationship: + source: zabbix_alarm + relationship_type: on + target: host + template_id : zabbix_alarm_on_host +scenarios: + - scenario: + condition: host_contains_instance + actions: + - action: + action_type: raise_alarm + action_target: + target: instance + properties: + alarm_name: mock_datasource.3rd_degree_scenarios.deduced.alarm1 + severity: SEVERE + - scenario: + condition: host_contains_instance and deduced_alarm_on_instance + actions: + - action: + action_type: raise_alarm + action_target: + target: instance + properties: + alarm_name: mock_datasource.3rd_degree_scenarios.deduced.alarm2 + severity: CRITICAL + - scenario: + condition: host_contains_instance and deduced_alarm_on_instance and second_iteration_deduced_alarm_on_instance + actions: + - action: + action_type: add_causal_relationship + action_target: + source: deduced_alarm + target: second_iteration_deduced_alarm + - scenario: + condition: zabbix_alarm_on_host and host_contains_instance and deduced_alarm_on_instance and second_iteration_deduced_alarm_on_instance + actions: + - action: + action_type: add_causal_relationship + action_target: + source: second_iteration_deduced_alarm + target: zabbix_alarm