openstack/vitrage
Code Issues Proposed changes

Merge "support adding and deleting templates without the need to restart vitrage."

Browse Source
This commit is contained in:
Zuul 2018-01-18 21:23:24 +00:00 committed by Gerrit Code Review
commit da7bc6e0c8
43 changed files with 1348 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
devstack/post_test_hook.sh
View File

@ -24,22 +24,25 @@ else
TESTS="topology"
fi
if [ "$DEVSTACK_GATE_USE_PYTHON3" == "True" ]; then
export PYTHON=python3
fi
sudo cp -rf $DEVSTACK_PATH/vitrage/vitrage_tempest_tests/tests/resources/static_physical/static_physical_configuration.yaml /etc/vitrage/
sudo cp -rf $DEVSTACK_PATH/vitrage/vitrage_tempest_tests/tests/resources/heat/heat_template.yaml /etc/vitrage/
sudo cp -rf $DEVSTACK_PATH/vitrage/vitrage_tempest_tests/tests/resources/heat/heat_nested_template.yaml /etc/vitrage/
sudo cp -rf $DEVSTACK_PATH/vitrage/vitrage_tempest_tests/tests/resources/heat/server.yaml /etc/vitrage/
sudo cp -rf $DEVSTACK_PATH/vitrage/vitrage_tempest_tests/tests/resources/templates/api/* /etc/vitrage/templates/
sudo cp $DEVSTACK_PATH/tempest/etc/logging.conf.sample $DEVSTACK_PATH/tempest/etc/logging.conf
# copied the templates need to restart
${PYTHON:-python} $DEVSTACK_PATH/vitrage/vitrage_tempest_tests/add_legacy_dir_templates.py
# restart due to configuration files changes
sudo systemctl restart devstack@vitrage-graph.service
# wait for 30 seconds
sleep 30
if [ "$DEVSTACK_GATE_USE_PYTHON3" == "True" ]; then
export PYTHON=python3
fi
cd $DEVSTACK_PATH/tempest/; sudo -E testr init

104
vitrage/api/controllers/v1/template.py
View File

@ -21,7 +21,9 @@ from pecan.core import abort
from vitrage.api.controllers.rest import RootRestController
from vitrage.api.policy import enforce
from vitrage.common.constants import TemplateStatus as TStatus
from vitrage.common.exception import VitrageError
from vitrage.evaluator.template_db import template_repository as template_repo
LOG = log.getLogger(__name__)
@ -65,6 +67,38 @@ class TemplateController(RootRestController):
to_unicode)
abort(404, to_unicode)
@pecan.expose('json')
def delete(self, **kwargs):
uuid = kwargs['uuid']
LOG.info("delete template. uuid: %s", str(uuid))
enforce("template delete",
pecan.request.headers,
pecan.request.enforcer,
{})
try:
return self._delete(uuid)
except Exception as e:
LOG.exception('failed to delete template %s', e)
abort(404, str(e))
@pecan.expose('json')
def put(self, **kwargs):
template_path = kwargs['path']
LOG.info("add template: %s", template_path)
enforce("template add",
pecan.request.headers,
pecan.request.enforcer,
{})
template_type = kwargs['template_type']
try:
return self._add(template_path, template_type)
except Exception as e:
LOG.exception('failed to add template %s', e)
abort(404, str(e))
@pecan.expose('json')
def post(self, **kwargs):
@ -84,30 +118,25 @@ class TemplateController(RootRestController):
LOG.exception('failed to validate template(s) %s', to_unicode)
abort(404, to_unicode)
@staticmethod
def _get_templates():
templates_json = pecan.request.client.call(pecan.request.context,
'get_templates')
LOG.info(templates_json)
@classmethod
def _get_templates(cls):
try:
template_list = json.loads(templates_json)['templates_details']
return template_list
templates = pecan.request.storage.templates.query()
templates = [t for t in templates if t.status != TStatus.DELETED]
templates.sort(key=lambda template: template.created_at)
return [cls._db_template_to_dict(t) for t in templates]
except Exception as e:
to_unicode = encodeutils.exception_to_unicode(e)
LOG.exception('failed to get template list %s ', to_unicode)
abort(404, to_unicode)
@staticmethod
def _show_template(template_uuid):
template_json = pecan.request.client.call(pecan.request.context,
'show_template',
template_uuid=template_uuid)
LOG.info(template_json)
def _show_template(uuid):
try:
return json.loads(template_json)
templates = pecan.request.storage.templates.query(uuid=uuid)
if not templates:
raise VitrageError("Template %s not found", uuid)
return templates[0].file_content
except Exception as e:
to_unicode = encodeutils.exception_to_unicode(e)
LOG.exception('failed to show template with uuid: %s ', to_unicode)
@ -125,3 +154,44 @@ class TemplateController(RootRestController):
to_unicode = encodeutils.exception_to_unicode(e)
LOG.exception('failed to open template file(s) %s ', to_unicode)
abort(404, to_unicode)
@classmethod
def _add(cls, path, template_type):
try:
templates = template_repo.add_template_to_db(
pecan.request.storage, path, template_type)
pecan.request.client.call(pecan.request.context, 'add_template')
return [cls._db_template_to_dict(t) for t in templates]
except Exception as e:
LOG.exception('failed to add template file %s ', e)
abort(404, str(e))
@classmethod
def _db_template_to_dict(cls, template):
return {
"uuid": template.uuid,
"name": template.name,
"status": template.status,
"date": template.created_at,
"status details": template.status_details,
"type": template.template_type,
}
@staticmethod
def _delete(uuid):
try:
storage = pecan.request.storage
templates = storage.templates.query(uuid=uuid)
if not templates:
raise VitrageError('template does not exists')
elif templates[0].status == TStatus.DELETED:
raise VitrageError('template is deleted')
elif templates[0].status == TStatus.ERROR:
storage.templates.update(uuid, "status", TStatus.DELETED)
elif templates[0].status == TStatus.ACTIVE:
storage.templates.update(uuid, "status", TStatus.DELETING)
pecan.request.client.call(pecan.request.context,
'delete_template')
except Exception as e:
LOG.exception('failed to delete template file %s ', e)
abort(404, str(e))

55
vitrage/api_handler/apis/template.py
View File

@ -16,7 +16,6 @@ import json
from oslo_log import log
from osprofiler import profiler
from vitrage.evaluator.template_fields import TemplateFields
from vitrage.evaluator.template_validation.content.template_content_validator \
import content_validation
from vitrage.evaluator.template_validation.status_messages import status_msgs
@ -34,41 +33,8 @@ class TemplateApis(object):
FAILED_MSG = 'validation failed'
OK_MSG = 'validation OK'
def __init__(self, templates, def_templates=None):
if def_templates is None:
def_templates = {}
self.def_templates = def_templates
self.templates = templates
def get_templates(self, ctx):
LOG.debug("TemplateApis get_templates")
templates_details = []
for uuid, template in self.templates.items():
template_metadata = template.data[TemplateFields.METADATA]
templates_details.append({
'uuid': str(template.uuid),
'name': template_metadata[TemplateFields.NAME],
'status': self._get_template_status(template.result),
'status details': template.result.comment,
'date': template.date.strftime('%Y-%m-%dT%H:%M:%SZ')
})
return json.dumps({'templates_details': templates_details})
def show_template(self, ctx, template_uuid):
LOG.debug("Show template with uuid: %s", str(template_uuid))
template = self.templates[template_uuid]
if template:
return json.dumps(template.data)
else:
return json.dumps({'ERROR': 'Incorrect uuid'})
def __init__(self, notifier=None):
self.notifier = notifier
def validate_template(self, ctx, templates):
LOG.debug("TemplateApis validate_template templates:"
@ -111,6 +77,23 @@ class TemplateApis(object):
return json.dumps({'results': results})
def add_template(self, ctx):
"""Signal the evaluator
A new template has been added to the database with a status of
LOADING that needs to be handled.
"""
LOG.info("Add Template Running")
self.notifier.notify("add template", {'template_action': 'add'})
def delete_template(self, ctx):
"""Signal the evaluator
A template status has been changed to DELETING.
"""
LOG.info("Delete Template Running")
self.notifier.notify("delete template", {'template_action': 'delete'})
@staticmethod
def _add_result(template_path, status, description, message, status_code,
results):

12
vitrage/api_handler/service.py
View File

@ -16,6 +16,9 @@ from oslo_log import log
import oslo_messaging
from oslo_service import service as os_service
from vitrage.entity_graph import EVALUATOR_TOPIC
from vitrage.messaging import VitrageNotifier
from vitrage.api_handler.apis.alarm import AlarmApis
from vitrage.api_handler.apis.event import EventApis
from vitrage.api_handler.apis.rca import RcaApis
@ -30,11 +33,12 @@ LOG = log.getLogger(__name__)
class VitrageApiHandlerService(os_service.Service):
def __init__(self, conf, e_graph, scenario_repo):
def __init__(self, conf, e_graph):
super(VitrageApiHandlerService, self).__init__()
self.conf = conf
self.entity_graph = e_graph
self.scenario_repo = scenario_repo
self.notifier = VitrageNotifier(self.conf, "vitrage.api",
EVALUATOR_TOPIC)
def start(self):
LOG.info("Vitrage Api Handler Service - Starting...")
@ -49,9 +53,7 @@ class VitrageApiHandlerService(os_service.Service):
endpoints = [TopologyApis(self.entity_graph, self.conf),
AlarmApis(self.entity_graph, self.conf),
RcaApis(self.entity_graph, self.conf),
TemplateApis(
self.scenario_repo.templates,
self.scenario_repo.def_templates),
TemplateApis(self.notifier),
EventApis(self.conf),
ResourceApis(self.entity_graph, self.conf)]

23
vitrage/cli/graph.py
View File

@ -21,11 +21,12 @@ from vitrage.api_handler.service import VitrageApiHandlerService
from vitrage.cli import VITRAGE_TITLE
from vitrage import entity_graph
from vitrage.entity_graph.consistency.service import VitrageConsistencyService
from vitrage.entity_graph.service import VitrageGraphService
from vitrage.evaluator.scenario_repository import ScenarioRepository
from vitrage import service
from vitrage import storage
from vitrage.entity_graph.service import VitrageGraphService
from vitrage.evaluator.evaluator_service import EvaluatorManager
def main():
"""Starts all the Entity graph services
@ -38,28 +39,28 @@ def main():
print(VITRAGE_TITLE)
conf = service.prepare_service()
e_graph = entity_graph.get_graph_driver(conf)('Entity Graph')
evaluator = EvaluatorManager(conf, e_graph)
launcher = os_service.ServiceLauncher(conf)
full_scenario_repo = ScenarioRepository(conf)
clear_db(conf)
db_connection = storage.get_connection_from_config(conf)
clear_active_actions_table(db_connection)
launcher.launch_service(VitrageGraphService(conf, e_graph))
launcher.launch_service(VitrageGraphService(
conf, e_graph, evaluator, db_connection))
launcher.launch_service(VitrageApiHandlerService(
conf, e_graph, full_scenario_repo))
launcher.launch_service(VitrageApiHandlerService(conf, e_graph))
launcher.launch_service(VitrageConsistencyService(conf, e_graph))
launcher.wait()
def clear_db(conf):
"""Delete all data from vitrage tables
def clear_active_actions_table(db_connection):
"""Delete all data from active_actions table
The following deletes the entire vitrage database
It should be removed once graph is persistent
"""
db_connection = storage.get_connection_from_config(conf)
db_connection.clear()
db_connection.active_actions.delete()
if __name__ == "__main__":
sys.exit(main())

8
vitrage/common/constants.py
View File

@ -161,3 +161,11 @@ class TemplateTypes(object):
STANDARD = 'standard'
DEFINITION = 'definition'
EQUIVALENCE = 'equivalence'
class TemplateStatus(object):
ACTIVE = 'ACTIVE'
ERROR = 'ERROR'
DELETING = 'DELETING'
DELETED = 'DELETED'
LOADING = 'LOADING'

22
vitrage/common/policies/template.py
View File

@ -17,6 +17,28 @@ from vitrage.common.policies import base
TEMPLATE = 'template %s'
rules = [
policy.DocumentedRuleDefault(
name=TEMPLATE % 'delete',
check_str=base.UNPROTECTED,
description='Delete a template',
operations=[
{
'path': '/template',
'method': 'DELETE'
}
]
),
policy.DocumentedRuleDefault(
name=TEMPLATE % 'add',
check_str=base.UNPROTECTED,
description='Add a template',
operations=[
{
'path': '/template',
'method': 'PUT'
}
]
),
policy.DocumentedRuleDefault(
name=TEMPLATE % 'validate',
check_str=base.UNPROTECTED,

21
vitrage/entity_graph/service.py
View File

@ -21,7 +21,7 @@ from oslo_service import service as os_service
from vitrage.entity_graph import EVALUATOR_TOPIC
from vitrage.entity_graph.processor.processor import Processor
from vitrage.entity_graph.vitrage_init import VitrageInit
from vitrage.evaluator.evaluator_service import EvaluatorManager
from vitrage.evaluator.template_loader_service import TemplateLoaderManager
from vitrage import messaging
from vitrage.persistency.graph_persistor import GraphPersistor
@ -32,12 +32,16 @@ class VitrageGraphService(os_service.Service):
def __init__(self,
conf,
graph):
graph,
evaluator,
db):
super(VitrageGraphService, self).__init__()
self.conf = conf
self.graph = graph
self.evaluator = EvaluatorManager(conf, graph)
self.init = VitrageInit(conf, graph, self.evaluator)
self.evaluator = evaluator
self.templates_loader = TemplateLoaderManager(conf, graph, db)
self.init = VitrageInit(conf, graph, self.evaluator,
self.templates_loader)
self.graph_persistor = GraphPersistor(conf) if \
self.conf.persistency.enable_persistency else None
self.processor = Processor(self.conf, self.init, graph,
@ -49,10 +53,17 @@ class VitrageGraphService(os_service.Service):
evaluator_topic = EVALUATOR_TOPIC
return TwoPriorityListener(
self.conf,
self.processor.process_event,
self.process_event,
collector_topic,
evaluator_topic)
def process_event(self, event):
if event.get('template_action'):
self.templates_loader.handle_template_event(event)
self.evaluator.reload_evaluators_templates()
else:
self.processor.process_event(event)
def start(self):
LOG.info("Vitrage Graph Service - Starting...")
super(VitrageGraphService, self).start()

5
vitrage/entity_graph/vitrage_init.py
View File

@ -24,10 +24,11 @@ class VitrageInit(object):
RECEIVED_ALL_END_MESSAGES = 'received_all_end_messages'
FINISHED = 'finished'
def __init__(self, conf, graph=None, evaluator=None):
def __init__(self, conf, graph=None, evaluator=None, template_loader=None):
self.conf = conf
self.graph = graph
self.evaluator = evaluator
self.template_loader = template_loader
self.status = self.STARTED
self.end_messages = {}
@ -44,6 +45,8 @@ class VitrageInit(object):
on_end_messages_func()
self.evaluator.start()
if self.template_loader:
self.template_loader.start()
# TODO(idan_hefetz) As vitrage is not yet persistent, there aren't
# TODO(idan_hefetz) any deduced alarms to be removed during init

1
vitrage/evaluator/__init__.py
View File

@ -16,7 +16,6 @@ from oslo_config import cfg
from vitrage.evaluator.template_schemas import init_template_schemas
# Register options for the service
OPTS = [
cfg.StrOpt('templates_dir',

2
vitrage/evaluator/base.py
View File

@ -14,7 +14,7 @@
from collections import namedtuple
import re
Template = namedtuple('Template', ['uuid', 'data', 'date', 'result'])
Template = namedtuple('Template', ['uuid', 'data', 'date'])
def is_function(str):

16
vitrage/evaluator/equivalence_repository.py
View File

@ -11,20 +11,26 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes
from vitrage.common.exception import VitrageError
from vitrage.evaluator.template_loading.equivalence_loader import \
EquivalenceLoader
from vitrage.utils import file as file_utils
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
class EquivalenceRepository(object):
def __init__(self):
self.entity_equivalences = {}
def load_files(self, directory):
equivalence_defs = file_utils.load_yaml_files(directory)
def load(self, db):
equivalence_defs = db.templates.query(
template_type=TemplateTypes.EQUIVALENCE,
status=TemplateStatus.ACTIVE)
equivalence_defs = [e.file_content for e in equivalence_defs]
for equivalence_def in equivalence_defs:
equivalences = EquivalenceLoader(equivalence_def).equivalences
for equivalence in equivalences:

6
vitrage/evaluator/evaluator_service.py
View File

@ -99,4 +99,8 @@ class EvaluatorWorker(base.GraphCloneWorkerBase):
self._reload_templates()
def _reload_templates(self):
raise NotImplementedError()
scenario_repo = ScenarioRepository(self._conf, self._worker_index,
self._workers_num)
self._evaluator.scenario_repo = scenario_repo
LOG.info("reloading evaluator scenarios")
self._evaluator.scenario_repo.log_enabled_scenarios()

12
vitrage/evaluator/scenario_evaluator.py
View File

@ -81,14 +81,18 @@ class ScenarioEvaluator(object):
def scenario_repo(self, scenario_repo):
self._scenario_repo = scenario_repo
def run_evaluator(self):
def run_evaluator(self, action_mode=ActionMode.DO):
self.enabled = True
vertices = self._entity_graph.get_vertices()
start_time = time.time()
for vertex in vertices:
self.process_event(None, vertex, True)
LOG.info('Run Evaluator on %s items - took %s', str(len(vertices)),
str(time.time() - start_time))
if action_mode == ActionMode.DO:
self.process_event(None, vertex, True)
elif action_mode == ActionMode.UNDO:
self.process_event(vertex, None, True)
LOG.info(
'Run %s Evaluator on %s items - took %s',
action_mode, str(len(vertices)), str(time.time() - start_time))
def process_event(self, before, current, is_vertex, *args, **kwargs):
"""Notification of a change in the entity graph.

122
vitrage/evaluator/scenario_repository.py
View File

@ -17,26 +17,18 @@ from collections import namedtuple
import itertools
from oslo_log import log
from oslo_utils import uuidutils
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.common.utils import get_portion
from vitrage.evaluator.base import Template
from vitrage.evaluator.equivalence_repository import EquivalenceRepository
from vitrage.evaluator.template_fields import TemplateFields
from vitrage.evaluator.template_loading.scenario_loader import ScenarioLoader
from vitrage.evaluator.template_loading.template_loader import TemplateLoader
from vitrage.evaluator.template_validation.content.base import \
get_template_schema
from vitrage.evaluator.template_validation.content.template_content_validator \
import content_validation
from vitrage.evaluator.template_validation.template_syntax_validator import \
def_template_syntax_validation
from vitrage.evaluator.template_validation.template_syntax_validator import \
EXCEPTION
from vitrage.evaluator.template_validation.template_syntax_validator import \
syntax_validation
from vitrage.graph.filter import check_filter as check_subset
from vitrage.utils import datetime as datetime_utils
from vitrage import storage
from vitrage.utils import file as file_utils
LOG = log.getLogger(__name__)
@ -56,12 +48,12 @@ class ScenarioRepository(object):
self._templates = {}
self._def_templates = {}
self._all_scenarios = []
self.entity_equivalences = EquivalenceRepository().load_files(
conf.evaluator.equivalences_dir)
self._db = storage.get_connection_from_config(conf)
self.entity_equivalences = EquivalenceRepository().load(self._db)
self.relationship_scenarios = defaultdict(list)
self.entity_scenarios = defaultdict(list)
self._load_def_template_files(conf)
self._load_templates_files(conf)
self._load_def_templates_from_db()
self._load_templates_from_db()
self._enable_worker_scenarios(worker_index, workers_num)
self.actions = self._create_actions_collection()
@ -108,57 +100,21 @@ class ScenarioRepository(object):
return scenarios
def add_template(self, template_def):
def _add_template(self, template):
self.templates[template.uuid] = Template(template.uuid,
template.file_content,
template.created_at)
template_data = TemplateLoader().load(template.file_content,
self._def_templates)
for scenario in template_data.scenarios:
for equivalent_scenario in self._expand_equivalence(scenario):
self._add_scenario(equivalent_scenario)
result = syntax_validation(template_def)
if not result.is_valid_config:
LOG.info('Unable to load template, syntax err: %s'
% result.comment)
else:
result = content_validation(template_def, self._def_templates)
if not result.is_valid_config:
LOG.info('Unable to load template, content err: %s'
% result.comment)
template_uuid = uuidutils.generate_uuid()
current_time = datetime_utils.utcnow()
self.templates[str(template_uuid)] = Template(template_uuid,
template_def,
current_time,
result)
if result.is_valid_config:
template_data = \
TemplateLoader().load(template_def, self._def_templates)
for scenario in template_data.scenarios:
for equivalent_scenario in self._expand_equivalence(scenario):
self._add_scenario(equivalent_scenario)
def add_def_template(self, def_template):
result, template_schema = get_template_schema(def_template)
if result.is_valid_config:
result = def_template_syntax_validation(def_template)
if not result.is_valid_config:
LOG.info('Unable to load definition template, syntax err: %s'
% result.comment)
if result.is_valid_config:
def_validator = \
template_schema.validators.get(TemplateFields.DEFINITIONS)
result = \
def_validator.def_template_content_validation(def_template)
if result.is_valid_config:
current_time = datetime_utils.utcnow()
include_uuid = uuidutils.generate_uuid()
self._def_templates[str(include_uuid)] = Template(include_uuid,
def_template,
current_time,
result)
else:
LOG.info('Unable to load definition template, content err: %s'
% result.comment)
def _add_def_template(self, def_template):
self.def_templates[def_template.uuid] = Template(
def_template.uuid,
def_template.file_content,
def_template.created_at)
def _expand_equivalence(self, scenario):
equivalent_scenarios = [scenario]
@ -191,29 +147,21 @@ class ScenarioRepository(object):
self._add_relationship_scenario(scenario, relationship)
self._all_scenarios.append(scenario)
def _load_def_template_files(self, conf):
def _load_def_templates_from_db(self):
def_templates = self._db.templates.query(
template_type=TType.DEFINITION,
status=TemplateStatus.ACTIVE)
for def_template in def_templates:
self._add_def_template(def_template)
if DEF_TEMPLATES_DIR_OPT in conf.evaluator:
def_templates_dir = conf.evaluator.def_templates_dir
def_templates = file_utils.load_yaml_files(def_templates_dir)
for def_template in def_templates:
self.add_def_template(def_template)
def _load_templates_files(self, conf):
templates_dir = conf.evaluator.templates_dir
files = \
file_utils.list_files(templates_dir, '.yaml', with_pathname=True)
template_defs = []
for f in files:
template_defs.append(self._load_template_file(f))
for template_def in template_defs:
self.add_template(template_def)
def _load_templates_from_db(self):
items = self._db.templates.query(template_type=TType.STANDARD)
# TODO(ikinory): statuses may cause loading templates to be running
templates = [x for x in items if x.status in [TemplateStatus.ACTIVE,
TemplateStatus.LOADING,
TemplateStatus.DELETING]]
for t in templates:
self._add_template(t)
@staticmethod
def _load_template_file(file_name):

15
vitrage/evaluator/template_db/__init__.py Normal file
View File

@ -0,0 +1,15 @@
# Copyright 2018 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
__author__ = 'stack'

126
vitrage/evaluator/template_db/template_repository.py Normal file
View File

@ -0,0 +1,126 @@
# Copyright 2018 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from oslo_log import log
from oslo_utils import uuidutils
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.common.exception import VitrageError
from vitrage.evaluator.base import Template
from vitrage.evaluator.template_fields import TemplateFields
from vitrage.evaluator import template_validation
from vitrage.evaluator.template_validation import base
from vitrage.evaluator.template_validation.template_syntax_validator import \
EXCEPTION
from vitrage.storage.sqlalchemy import models
from vitrage.utils import file
LOG = log.getLogger(__name__)
METADATA = 'metadata'
NAME = 'name'
def add_template_to_db(db, path, template_type):
"""Add templates to db
Loads template files, for every template, check it is valid and does
not exist, if so adds it to the database.
:param db:
:param path: path to a file or directory
:param template_type: standard/definition/equivalence
:return: all the templates that were added
:rtype: list of models.Template
"""
added_rows = list()
files = _list_files(path)
for f in files:
template = load_template_file(f)
template_name = template[METADATA][NAME]
templates = db.templates.query(name=template_name)
if [t for t in templates if t.status != TemplateStatus.DELETED]:
LOG.warning("Duplicate templates found %s."
" new template will not be added", template_name)
else:
validation_result = _validate_template(db, template, template_type)
db_row = _to_db_row(validation_result, template, template_type)
db.templates.create(db_row)
added_rows.append(db_row)
return added_rows
def _list_files(path):
if os.path.isdir(path):
LOG.info("Adding all templates from %s", path)
return file.list_files(path, '.yaml', with_pathname=True)
elif os.path.isfile(path):
LOG.info("Adding template %s", path)
return [path] # only one file
else:
raise VitrageError("No such file or directory %s" % path)
def _validate_template(db, template, template_type):
if template_type == TType.DEFINITION:
result = template_validation.validate_definition_template(template)
elif template_type == TType.STANDARD:
result = template_validation.validate_template(template,
_load_def_templates(db))
elif template_type == TType.EQUIVALENCE:
result = base.Result("", True, "", "No Validation")
else:
raise VitrageError("Unknown template type %s", template_type)
return result
def load_template_file(file_name):
try:
return file.load_yaml_file(file_name, with_exception=True)
except Exception as e:
return {TemplateFields.METADATA: {TemplateFields.NAME: file_name},
EXCEPTION: str(e)}
def _to_db_row(result, template, template_type):
uuid = uuidutils.generate_uuid()
status = TemplateStatus.LOADING if result.is_valid_config else \
TemplateStatus.ERROR
status_details = result.comment
db_row = models.Template(
name=template[METADATA][NAME],
uuid=uuid,
status=status,
status_details=status_details,
file_content=template,
template_type=template_type,
)
return db_row
def _load_def_templates(db):
def_templates = {}
items = db.templates.query(template_type=TType.DEFINITION)
def_templates_db = [x for x in items if x.status in [
TemplateStatus.ACTIVE,
TemplateStatus.LOADING]]
for df in def_templates_db:
def_templates[df.uuid] = Template(df.uuid,
df.file_content,
df.created_at)
return def_templates

120
vitrage/evaluator/template_loader_service.py Normal file
View File

@ -0,0 +1,120 @@
# Copyright 2017 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import multiprocessing
from oslo_log import log
from vitrage.common.constants import TemplateStatus as TStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.common.exception import VitrageError
from vitrage.entity_graph import EVALUATOR_TOPIC
from vitrage.entity_graph.graph_clone import base
from vitrage.evaluator.actions.base import ActionMode
from vitrage.evaluator.scenario_evaluator import ScenarioEvaluator
from vitrage.evaluator.scenario_repository import ScenarioRepository
from vitrage.messaging import VitrageNotifier
LOG = log.getLogger(__name__)
TEMPLATE_ACTION = 'template_action'
class TemplateLoaderManager(base.GraphCloneManagerBase):
def __init__(self, conf, entity_graph, db):
super(TemplateLoaderManager, self).__init__(conf, entity_graph, 1)
self._db = db
def _run_worker(self, worker_index, workers_num):
tasks_queue = multiprocessing.JoinableQueue()
w = TemplateLoaderWorker(
self._conf,
tasks_queue,
self._entity_graph)
self._p_launcher.launch_service(w)
return tasks_queue
def handle_template_event(self, event):
template_action = event.get('template_action')
if template_action == 'add':
templates = self._db.templates.query(status=TStatus.LOADING)
new_status = TStatus.ACTIVE
action_mode = ActionMode.DO
elif template_action == 'delete':
templates = self._db.templates.query(status=TStatus.DELETING)
new_status = TStatus.DELETED
action_mode = ActionMode.UNDO
else:
raise VitrageError('Invalid template_action %s' % template_action)
self._template_worker_task(
[t.name for t in templates if t.template_type == TType.STANDARD],
action_mode)
for t in templates:
self._db.templates.update(t.uuid, 'status', new_status)
def _template_worker_task(self, template_names, action_mode):
self._notify_and_wait((TEMPLATE_ACTION, template_names, action_mode))
class TemplateLoaderWorker(base.GraphCloneWorkerBase):
def __init__(self,
conf,
task_queue,
e_graph):
super(TemplateLoaderWorker, self).__init__(conf, task_queue, e_graph)
self._evaluator = None
def start(self):
super(TemplateLoaderWorker, self).start()
actions_callback = VitrageNotifier(
conf=self._conf,
publisher_id='vitrage_evaluator',
topic=EVALUATOR_TOPIC).notify
self._evaluator = ScenarioEvaluator(
self._conf,
self._entity_graph,
None,
actions_callback,
enabled=False)
def do_task(self, task):
super(TemplateLoaderWorker, self).do_task(task)
action = task[0]
if action == TEMPLATE_ACTION:
(action, template_names, action_mode) = task
self._template_action(template_names, action_mode)
def _template_action(self, template_names, action_mode):
self._enable_evaluator_templates(template_names)
self._evaluator.run_evaluator(action_mode)
self._disable_evaluator()
def _enable_evaluator_templates(self, template_names):
scenario_repo = ScenarioRepository(self._conf)
for s in scenario_repo._all_scenarios:
s.enabled = False
for template_name in template_names:
if s.id.startswith(template_name):
s.enabled = True
self._evaluator.scenario_repo = scenario_repo
self._evaluator.scenario_repo.log_enabled_scenarios()
self._evaluator.enabled = True
def _disable_evaluator(self):
self._entity_graph.notifier._subscriptions = [] # Quick n dirty
self._evaluator.enabled = False

44
vitrage/evaluator/template_validation/__init__.py
View File

@ -11,4 +11,48 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from vitrage.evaluator.template_fields import TemplateFields
from vitrage.evaluator.template_validation.content.base import \
get_template_schema
from vitrage.evaluator.template_validation.content.template_content_validator \
import content_validation
from vitrage.evaluator.template_validation.template_syntax_validator import \
def_template_syntax_validation
from vitrage.evaluator.template_validation.template_syntax_validator import \
syntax_validation
__author__ = 'stack'
LOG = log.getLogger(__name__)
def validate_template(template, def_templates):
result = syntax_validation(template)
if not result.is_valid_config:
LOG.error('Unable to load template, syntax err: %s' % result.comment)
return result
result = content_validation(template, def_templates)
if not result.is_valid_config:
LOG.error('Unable to load template, content err: %s' % result.comment)
return result
return result
def validate_definition_template(template_def):
result, template_schema = get_template_schema(template_def)
if not result.is_valid_config:
return result
result = def_template_syntax_validation(template_def)
if not result.is_valid_config:
LOG.error('Unable to load template, syntax err: %s' % result.comment)
return result
validator = template_schema.validators.get(TemplateFields.DEFINITIONS)
result = validator.def_template_content_validation(template_def)
if not result.is_valid_config:
LOG.error('Unable to load template, content err: %s' % result.comment)
return result
return result

2
vitrage/storage/base.py
View File

@ -121,7 +121,7 @@ class TemplatesConnection(object):
@abc.abstractmethod
def query(self, name=None, file_content=None,
uuid=None, status=None, status_details=None, is_deleted=None,
uuid=None, status=None, status_details=None,
template_type=None):
"""Yields a lists of templates that match filters.

3
vitrage/storage/impl_sqlalchemy.py
View File

@ -116,7 +116,7 @@ class TemplatesConnection(base.TemplatesConnection, BaseTableConn):
session.query(Template).filter_by(uuid=uuid).update({var: value})
def query(self, name=None, file_content=None,
uuid=None, status=None, status_details=None, is_deleted=None,
uuid=None, status=None, status_details=None,
template_type=None):
query = self.query_filter(
models.Template,
@ -125,7 +125,6 @@ class TemplatesConnection(base.TemplatesConnection, BaseTableConn):
uuid=uuid,
status=status,
status_details=status_details,
is_deleted=is_deleted,
template_type=template_type,
)
return query.all()

6
vitrage/storage/sqlalchemy/models.py
View File

@ -16,7 +16,7 @@ import json
from oslo_db.sqlalchemy import models
from sqlalchemy import Column, DateTime, INTEGER, String, \
SmallInteger, BigInteger, Index, Boolean
SmallInteger, BigInteger, Index
from sqlalchemy.ext.declarative import declarative_base
import sqlalchemy.types as types
@ -146,13 +146,12 @@ class Template(Base, models.TimestampMixin):
status_details = Column(String(128))
name = Column(String(128), nullable=False)
file_content = Column(JSONEncodedDict, nullable=False)
is_deleted = Column(Boolean, nullable=False, default=False)
template_type = Column("type", String(64), default='standard')
def __repr__(self):
return "<Template(id='%s', name='%s', created_at='%s'," \
" updated_at='%s', status='%s'," \
"status_details='%s', file_content='%s', is_deleted='%s'," \
"status_details='%s', file_content='%s', " \
" template_type='%s' )>" % \
(self.uuid,
self.name,
@ -161,5 +160,4 @@ class Template(Base, models.TimestampMixin):
self.status,
self.status_details,
self.file_content,
self.is_deleted,
self.template_type,)

2
vitrage/tests/functional/api/__init__.py
View File

@ -55,7 +55,7 @@ class FunctionalTest(base.BaseTest):
self.CONF.set_override('auth_mode', self.auth, group='api')
self.CONF.set_override('connection',
'sqlite:///:memory:',
'sqlite:///:test.db:',
group='database')
self.app = webtest.TestApp(app.load_app(self.CONF))

10
vitrage/tests/functional/api/v1/test_noauth.py
View File

@ -19,6 +19,7 @@ from datetime import datetime
# noinspection PyPackageRequirements
from mock import mock
from vitrage.storage.sqlalchemy import models
from vitrage.tests.functional.api.v1 import FunctionalTest
@ -113,19 +114,20 @@ class NoAuthTest(FunctionalTest):
def test_noauth_mode_list_templates(self):
with mock.patch('pecan.request') as request:
request.client.call.return_value = '{"templates_details": []}'
request.storage.templates.query.return_value = []
data = self.get_json('/template/')
self.assertEqual(1, request.client.call.call_count)
self.assertEqual(1, request.storage.templates.query.call_count)
self.assertEqual([], data)
def test_noauth_mode_show_template(self):
with mock.patch('pecan.request') as request:
request.client.call.return_value = '{}'
request.storage.templates.query.return_value = \
[models.Template(file_content={})]
data = self.get_json('/template/1234')
self.assertEqual(1, request.client.call.call_count)
self.assertEqual(1, request.storage.templates.query.call_count)
self.assertEqual({}, data)
def test_noauth_mode_validate_template(self):

3
vitrage/tests/functional/entity_graph/consistency/test_consistency.py
View File

@ -13,11 +13,12 @@
# under the License.
from datetime import timedelta
from six.moves import queue
import time
import unittest
from oslo_config import cfg
from six.moves import queue
from vitrage.common.constants import EdgeLabel
from vitrage.common.constants import EntityCategory

3
vitrage/tests/functional/evaluator/test_action_executor.py
View File

@ -12,9 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
from six.moves import queue
from oslo_config import cfg
from six.moves import queue
from vitrage.common.constants import DatasourceAction
from vitrage.common.constants import DatasourceProperties as DSProp

21
vitrage/tests/functional/evaluator/test_scenario_evaluator.py
View File

@ -12,9 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from vitrage.tests.functional.test_configuration import TestConfiguration
LOG = log.getLogger(__name__)
from oslo_db.options import database_opts
from six.moves import queue
from oslo_config import cfg
@ -42,8 +44,6 @@ from vitrage.evaluator.actions.evaluator_event_transformer \
from vitrage.evaluator.scenario_evaluator import ScenarioEvaluator
from vitrage.evaluator.scenario_repository import ScenarioRepository
from vitrage.graph import create_edge
from vitrage import storage
from vitrage.storage.sqlalchemy import models
from vitrage.tests.functional.base import \
TestFunctionalBase
import vitrage.tests.mocks.mock_driver as mock_driver
@ -57,17 +57,13 @@ _NAGIOS_TEST_INFO = {NagiosProperties.RESOURCE_NAME: _TARGET_HOST,
DSProps.DATASOURCE_ACTION: DatasourceAction.SNAPSHOT}
class TestScenarioEvaluator(TestFunctionalBase):
class TestScenarioEvaluator(TestFunctionalBase, TestConfiguration):
EVALUATOR_OPTS = [
cfg.StrOpt('templates_dir',
default=utils.get_resources_dir() +
'/templates/evaluator',
),
cfg.StrOpt('equivalences_dir',
default=utils.get_resources_dir() +
'/equivalences',
),
cfg.StrOpt('notifier_topic',
default='vitrage.evaluator',
),
@ -81,13 +77,8 @@ class TestScenarioEvaluator(TestFunctionalBase):
cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
cls.conf.register_opts(cls.EVALUATOR_OPTS, group='evaluator')
cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
cls.conf.register_opts(database_opts, group='database')
cls.conf.set_override('connection', 'sqlite:///test.db',
group='database')
cls._db = storage.get_connection_from_config(cls.conf)
engine = cls._db._engine_facade.get_engine()
models.Base.metadata.create_all(engine)
cls.add_db(cls.conf)
cls.add_templates(cls.conf.evaluator.templates_dir)
TestScenarioEvaluator.load_datasources(cls.conf)
cls.scenario_repository = ScenarioRepository(cls.conf)

17
vitrage/tests/functional/test_configuration.py
View File

@ -13,10 +13,13 @@
# under the License.
from oslo_db.options import database_opts
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.evaluator.template_db.template_repository import \
add_template_to_db
from vitrage import storage
from vitrage.storage.sqlalchemy import models
TEMPLATE_DIR = '/etc/vitrage/templates'
@ -32,3 +35,15 @@ class TestConfiguration(object):
models.Base.metadata.drop_all(engine)
models.Base.metadata.create_all(engine)
return cls._db
@classmethod
def add_templates(cls, templates_dir, templates_type=TType.STANDARD):
templates = add_template_to_db(cls._db, templates_dir, templates_type)
for t in templates:
if t.status == TemplateStatus.LOADING:
cls._db.templates.update(t.uuid, 'status',
TemplateStatus.ACTIVE)
if t.status == TemplateStatus.DELETING:
cls._db.templates.update(t.uuid, 'status',
TemplateStatus.DELETED)
return templates

16
vitrage/tests/unit/evaluator/test_equivalence_repository.py
View File

@ -14,22 +14,30 @@
import os
from oslo_config import cfg
from vitrage.common.constants import TemplateTypes
from vitrage.common.exception import VitrageError
from vitrage.evaluator.equivalence_repository import EquivalenceRepository
from vitrage.tests import base
from vitrage.tests.functional.test_configuration import TestConfiguration
from vitrage.tests.mocks import utils
class TestEquivalenceRepository(base.BaseTest):
class TestEquivalenceRepository(base.BaseTest, TestConfiguration):
# noinspection PyPep8Naming
def setUp(self):
super(TestEquivalenceRepository, self).setUp()
conf = cfg.ConfigOpts()
self.add_db(conf)
self.equivalence_repository = EquivalenceRepository()
def test_duplicate_entities_in_equivalence(self):
base_dir = utils.get_resources_dir() + '/templates/equivalences_dup'
for directory in os.listdir(base_dir):
self.assertRaises(VitrageError,
self.equivalence_repository.load_files,
os.path.join(base_dir, directory))
self.add_templates(os.path.join(base_dir, directory),
TemplateTypes.EQUIVALENCE)
self.assertRaises(
VitrageError,
self.equivalence_repository.load, self._db)

40
vitrage/tests/unit/evaluator/test_scenario_repository.py
View File

@ -12,22 +12,22 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from oslo_config import cfg
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import TemplateTypes as TType
from vitrage.common.constants import VertexProperties as VProps
from vitrage.evaluator.scenario_repository import ScenarioRepository
from vitrage.evaluator.template_validation.template_syntax_validator import \
syntax_validation
from vitrage.graph import Vertex
from vitrage.tests import base
from vitrage.tests.functional.test_configuration import TestConfiguration
from vitrage.tests.mocks import utils
from vitrage.utils import file as file_utils
class ScenarioRepositoryTest(base.BaseTest):
class ScenarioRepositoryTest(base.BaseTest, TestConfiguration):
BASE_DIR = utils.get_resources_dir() + '/templates/general'
OPTS = [
cfg.StrOpt('templates_dir',
@ -44,7 +44,8 @@ class ScenarioRepositoryTest(base.BaseTest):
super(ScenarioRepositoryTest, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group='evaluator')
cls.add_db(cls.conf)
cls.add_templates(cls.conf.evaluator.templates_dir)
templates_dir_path = cls.conf.evaluator.templates_dir
cls.template_defs = file_utils.load_yaml_files(templates_dir_path)
@ -57,8 +58,10 @@ class ScenarioRepositoryTest(base.BaseTest):
# Test assertions
self.assertIsNotNone(scenario_repository)
path, dirs, files = next(os.walk(self.conf.evaluator.templates_dir))
self.assertEqual(len(files), len(scenario_repository.templates))
self.assertEqual(
2,
len(scenario_repository.templates),
'scenario_repository.templates should contain all valid templates')
def test_init_scenario_repository(self):
@ -74,7 +77,10 @@ class ScenarioRepositoryTest(base.BaseTest):
scenario_templates = self.scenario_repository.templates
# there is one bad template
self.assertEqual(valid_template_counter, len(scenario_templates) - 1)
self.assertEqual(
valid_template_counter,
len(scenario_templates),
'scenario_repository.templates should contain all valid templates')
entity_equivalences = self.scenario_repository.entity_equivalences
for entity_props, equivalence in entity_equivalences.items():
@ -100,20 +106,20 @@ class ScenarioRepositoryTest(base.BaseTest):
pass
class RegExTemplateTest(base.BaseTest):
class RegExTemplateTest(base.BaseTest, TestConfiguration):
BASE_DIR = utils.get_resources_dir() + '/templates/regex'
OPTS = [
cfg.StrOpt('templates_dir',
default=BASE_DIR),
cfg.StrOpt('equivalences_dir',
default=BASE_DIR + '/equivalences')]
default=BASE_DIR)]
@classmethod
def setUpClass(cls):
super(RegExTemplateTest, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group='evaluator')
cls.add_db(cls.conf)
cls.add_templates(cls.conf.evaluator.templates_dir)
cls.scenario_repository = ScenarioRepository(cls.conf)
def test_basic_regex(self):
@ -169,7 +175,7 @@ class RegExTemplateTest(base.BaseTest):
self.assertEqual(0, len(relevant_scenarios))
class EquivalentScenarioTest(base.BaseTest):
class EquivalentScenarioTest(base.BaseTest, TestConfiguration):
BASE_DIR = utils.get_resources_dir() + '/templates/equivalent_scenarios/'
OPTS = [
cfg.StrOpt('templates_dir',
@ -188,10 +194,12 @@ class EquivalentScenarioTest(base.BaseTest):
super(EquivalentScenarioTest, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.OPTS, group='evaluator')
templates_dir_path = cls.conf.evaluator.templates_dir
cls.template_defs = file_utils.load_yaml_files(templates_dir_path)
cls.add_db(cls.conf)
cls.add_templates(cls.conf.evaluator.templates_dir)
cls.add_templates(cls.conf.evaluator.equivalences_dir,
TType.EQUIVALENCE)
cls.add_templates(cls.conf.evaluator.def_templates_dir,
TType.DEFINITION)
cls.scenario_repository = ScenarioRepository(cls.conf)
def test_expansion(self):

43
vitrage_tempest_tests/add_legacy_dir_templates.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright 2017 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TType
from vitrage.evaluator.template_db.template_repository import \
add_template_to_db
from vitrage import service
from vitrage import storage
from vitrage_tempest_tests.tests.common import general_utils
files = ['corrupted_template.yaml', 'e2e_test_basic_actions.yaml',
'e2e_test_overlapping_actions.yaml', 'v1_execute_mistral.yaml',
'v2_execute_mistral.yaml', 'host_aodh_alarm_for_rca.yaml',
'nagios_alarm_for_alarms.yaml'
]
def main():
resources_path = general_utils.tempest_resources_dir() + '/templates/api/'
conf = service.prepare_service()
db = storage.get_connection_from_config(conf)
for f in files:
full_path = resources_path + f
template = add_template_to_db(db, full_path, TType.STANDARD)
if template[0]['name'] != 'corrupted_template':
db.templates.update(template[0]['uuid'],
'status', TemplateStatus.ACTIVE)
if __name__ == "__main__":
sys.exit(main())

11
vitrage_tempest_tests/tests/api/templates/base.py
View File

@ -15,10 +15,13 @@ import json
from oslo_log import log as logging
from vitrage.common.exception import VitrageError
from vitrage_tempest_tests.tests.base import BaseVitrageTempest
from vitrage_tempest_tests.tests.common import general_utils as g_utils
from vitrage_tempest_tests.tests.common import vitrage_utils
from vitrage_tempest_tests.tests import utils
LOG = logging.getLogger(__name__)
@ -162,3 +165,11 @@ class BaseTemplateTest(BaseVitrageTempest):
relationships, len(template_show['definitions']['relationships']))
self.assertEqual(
scenarios, len(template_show['scenarios']))
def _rollback_to_default(self, templates):
try:
for t in templates:
db_row = vitrage_utils.get_first_template(name=t)
vitrage_utils.delete_template(db_row['uuid'])
except Exception as e:
raise VitrageError('Rollback to default failed %s', e)

184
vitrage_tempest_tests/tests/api/templates/test_template.py
View File

@ -12,15 +12,30 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from vitrage_tempest_tests.tests.api.templates.base import BaseTemplateTest
import vitrage_tempest_tests.tests.utils as utils
import unittest
from oslo_log import log as logging
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes as TTypes
from vitrage.evaluator.template_db.template_repository import \
load_template_file
from vitrage_tempest_tests.tests.api.templates.base import BaseTemplateTest
from vitrage_tempest_tests.tests.common import general_utils as g_utils
from vitrage_tempest_tests.tests.common.tempest_clients import TempestClients
from vitrage_tempest_tests.tests.common import vitrage_utils
import vitrage_tempest_tests.tests.utils as utils
from vitrageclient.exceptions import ClientException
LOG = logging.getLogger(__name__)
STANDARD_TEMPLATE = 'host_high_memory_usage_scenarios.yaml'
EQUIVALENCE_TEMPLATE = 'basic_equivalence_templates.yaml'
DEFINITION_TEMPLATE = 'basic_def_template.yaml'
STANDARD_ERROR = 'corrupted_template.yaml'
FAKE_UUID = 'ade68276-0fe9-42cd-9ec2-e7f20470a771'
class TestValidate(BaseTemplateTest):
"""Template test class for Vitrage API tests."""
@ -132,3 +147,162 @@ class TestValidate(BaseTemplateTest):
self._compare_template_show(
api_template_show, cli_template_show)
self._validate_template_structure(item, api_template_show)
class TemplatesDBTest(BaseTemplateTest):
"""Template DB test class for vitrage API tests"""
@classmethod
def setUpClass(cls):
super(TemplatesDBTest, cls).setUpClass()
cls.client = TempestClients.vitrage()
def test_template_add(self):
"""template add test
test standard , definition and equivalence templates
"""
templates_names = list()
try:
# TODO(ikinory): add folder of templates
# Add standard ,equivalence and definition templates
templates_names = self._add_templates()
vitrage_utils.add_template(STANDARD_TEMPLATE,
template_type=TTypes.STANDARD)
# assert standard template
db_row = vitrage_utils.get_first_template(
name='host_high_memory_usage_scenarios', type=TTypes.STANDARD)
self.assertEqual(db_row['name'],
'host_high_memory_usage_scenarios',
'standard template not found in list')
# assert equivalence template
db_row = vitrage_utils.get_first_template(
name='entity equivalence example',
type=TTypes.EQUIVALENCE)
self.assertEqual(db_row['name'],
'entity equivalence example',
'equivalence template not found in list')
# assert definition template
db_row = vitrage_utils.get_first_template(
name='basic_def_template',
type=TTypes.DEFINITION,
status=TemplateStatus.ACTIVE)
self.assertEqual(db_row['name'],
'basic_def_template',
'definition template not found in list')
# assert corrupted template - validate failed
db_row = vitrage_utils.get_first_template(
name='corrupted_template',
type=TTypes.STANDARD,
status=TemplateStatus.ERROR)
self.assertIsNotNone(
db_row,
'corrupted template template presented in list')
except Exception as e:
self._handle_exception(e)
raise
finally:
self._rollback_to_default(templates_names)
def test_template_delete(self):
try:
# add standard template
vitrage_utils.add_template(STANDARD_TEMPLATE,
template_type=TTypes.STANDARD)
db_row = vitrage_utils.get_first_template(
name='host_high_memory_usage_scenarios',
type=TTypes.STANDARD,
status=TemplateStatus.ACTIVE)
self.assertIsNotNone(db_row,
'Template should appear in templates list')
# delete template
uuid = db_row['uuid']
vitrage_utils.delete_template(uuid)
db_row = vitrage_utils.get_first_template(
name='host_high_memory_usage_scenarios', type=TTypes.STANDARD)
self.assertIsNone(db_row, 'Template should not appear in list')
# delete the same template again - should raise VitrageError
self.assertRaises(ClientException,
vitrage_utils.delete_template, uuid)
# delete non-existing template - should raise VitrageError
self.assertRaises(ClientException,
vitrage_utils.delete_template, FAKE_UUID)
except Exception as e:
self._handle_exception(e)
raise
def test_compare_cli_to_api(self):
"""Compare between api template list
to cli template list
compares each template in list
"""
templates_names = list()
try:
# Add standard ,equivalence and definition templates
templates_names = self._add_templates()
cli_templates_list = utils.run_vitrage_command(
"vitrage template list", self.conf)
api_templates_list = self.client.template.list()
self.assertNotEqual(len(api_templates_list), 0,
'The template list taken from api is empty')
self.assertIsNotNone(cli_templates_list,
'The template list taken from cli is empty')
self._validate_templates_list_length(api_templates_list,
cli_templates_list)
self._validate_passed_templates_length(api_templates_list,
cli_templates_list)
self._compare_each_template_in_list(api_templates_list,
cli_templates_list)
except Exception as e:
self._handle_exception(e)
raise
finally:
self._rollback_to_default(templates_names)
def test_template_show(self):
"""Compare template content from file to DB"""
try:
# add standard template
template_path = \
g_utils.tempest_resources_dir() + '/templates/api/'\
+ STANDARD_TEMPLATE
vitrage_utils.add_template(STANDARD_TEMPLATE,
template_type=TTypes.STANDARD)
db_row = vitrage_utils.get_first_template(
name='host_high_memory_usage_scenarios',
type=TTypes.STANDARD,
status=TemplateStatus.ACTIVE)
payload_from_db = self.client.template.show(db_row['uuid'])
payload_from_file = load_template_file(template_path)
self.assertEqual(payload_from_file, payload_from_db,
"Template content doesn't match")
vitrage_utils.delete_template(db_row['uuid'])
except Exception as e:
self._handle_exception(e)
raise
def _add_templates(self):
vitrage_utils.add_template(STANDARD_TEMPLATE,
template_type=TTypes.STANDARD)
vitrage_utils.add_template(EQUIVALENCE_TEMPLATE,
template_type=TTypes.EQUIVALENCE)
vitrage_utils.add_template(DEFINITION_TEMPLATE,
template_type=TTypes.DEFINITION)
vitrage_utils.add_template(STANDARD_ERROR,
template_type=TTypes.STANDARD)
return ['host_high_memory_usage_scenarios',
'entity equivalence example',
'basic_def_template',
'corrupted_template']

5
vitrage_tempest_tests/tests/common/general_utils.py
View File

@ -11,6 +11,7 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from os import path
import six
@ -37,3 +38,7 @@ def is_subset(subset, full):
def _remove_none_values(**kwargs):
return {k: v for k, v in kwargs.items() if v is not None}
def tempest_resources_dir():
return path.join(path.dirname(path.dirname(__file__)), 'resources')

42
vitrage_tempest_tests/tests/common/vitrage_utils.py
View File

@ -12,11 +12,17 @@
# License for the specific language governing permissions and limitations
# under the License.
from datetime import datetime
from oslo_log import log as logging
from vitrage.common.constants import TemplateStatus
from vitrage.common.constants import TemplateTypes
from vitrage.datasources import NOVA_HOST_DATASOURCE
from vitrage.datasources import NOVA_INSTANCE_DATASOURCE
from vitrage_tempest_tests.tests.common import general_utils as g_utils
from vitrage_tempest_tests.tests.common.tempest_clients import TempestClients
from vitrage_tempest_tests.tests.utils import wait_for_status
LOG = logging.getLogger(__name__)
DOWN = 'down'
UP = 'up'
@ -38,8 +44,13 @@ def generate_fake_host_alarm(hostname, event_type, enabled=True):
def get_first_host(**kwargs):
hosts = TempestClients.vitrage().resource.list(
NOVA_HOST_DATASOURCE, all_tenants=True)
try:
hosts = TempestClients.vitrage().resource.list(
NOVA_HOST_DATASOURCE, all_tenants=True)
except Exception as e:
LOG.exception("get_first_host failed with %s", e)
hosts = TempestClients.vitrage().resource.list(
NOVA_HOST_DATASOURCE, all_tenants=True)
return g_utils.first_match(hosts, **kwargs)
@ -47,3 +58,30 @@ def get_first_instance(**kwargs):
instances = TempestClients.vitrage().resource.list(
NOVA_INSTANCE_DATASOURCE, all_tenants=True)
return g_utils.first_match(instances, **kwargs)
def add_template(filename='',
folder='templates/api',
template_type=TemplateTypes.STANDARD):
full_path = g_utils.tempest_resources_dir() + '/' + folder + '/' + filename
t = TempestClients.vitrage().template.add(full_path, template_type)
if t and t[0]:
wait_for_status(
10,
get_first_template,
uuid=t[0]['uuid'], status=TemplateStatus.ACTIVE)
return t[0]
return None
def get_first_template(**kwargs):
templates = TempestClients.vitrage().template.list()
return g_utils.first_match(templates, **kwargs)
def delete_template(uuid):
TempestClients.vitrage().template.delete(uuid)
wait_for_status(
10,
lambda _id: True if not get_first_template(uuid=_id) else False,
_id=uuid)

189
vitrage_tempest_tests/tests/e2e/test_basic_template_actions.py Normal file
View File

@ -0,0 +1,189 @@
# Copyright 2017 - Nokia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
import time
from vitrage.common.constants import EntityCategory
from vitrage.common.constants import VertexProperties as VProps
from vitrage.evaluator.actions.evaluator_event_transformer import \
VITRAGE_DATASOURCE
from vitrage_tempest_tests.tests.common import vitrage_utils as v_util
from vitrage_tempest_tests.tests.e2e.test_actions_base import TestActionsBase
from vitrage_tempest_tests.tests import utils
LOG = logging.getLogger(__name__)
TRIGGER_ALARM_1 = 'e2e.test_template_actions.trigger.alarm1'
DEDUCED = 'e2e.test_template_actions.deduced.alarm'
TRIGGER_ALARM_2 = 'e2e.test_template_actions.trigger.alarm2'
DEDUCED_2 = 'e2e.test_template_actions.deduced.alarm2'
TEST_TEMPLATE = 'e2e_test_template_actions.yaml'
TEST_TEMPLATE_2 = 'e2e_test_template_actions_2.yaml'
INFILE_NAME = 'e2e_test_template_actions'
INFILE_NAME_2 = 'e2e_test_template_actions_2'
FOLDER_PATH = 'templates/api/e2e_test_template'
DEDUCED_PROPS = {
VProps.NAME: DEDUCED,
VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
VProps.VITRAGE_TYPE: VITRAGE_DATASOURCE,
}
DEDUCED_PROPS_2 = {
VProps.NAME: DEDUCED_2,
VProps.VITRAGE_CATEGORY: EntityCategory.ALARM,
VProps.VITRAGE_TYPE: VITRAGE_DATASOURCE,
}
class TestTemplateActions(TestActionsBase):
def __init__(self, *args, **kwds):
super(TestTemplateActions, self).__init__(*args, **kwds)
self.added_template = None
def setUp(self):
super(TestTemplateActions, self).setUp()
def tearDown(self):
super(TestTemplateActions, self).tearDown()
time.sleep(10)
self._trigger_undo_action(TRIGGER_ALARM_1)
if self.added_template is not None:
v_util.delete_template(self.added_template['uuid'])
self.added_template = None
@utils.tempest_logger
def test_evaluator_reload_with_new_template(self):
"""Test reload new template e2e
1. raise trigger alarm (template is not loaded yet)
2. add the relevant template
3. check action is executed
This checks that the evaluators are reloaded and run on all existing
vertices.
"""
try:
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self._trigger_do_action(TRIGGER_ALARM_1)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
self._check_deduced(1, DEDUCED_PROPS, host_id)
except Exception as e:
self._handle_exception(e)
raise
@utils.tempest_logger
def test_evaluator_reload_with_existing_template(self):
"""Test reload new template e2e
1.add the relevant template
2.raise trigger alarm.
3. check action is executed
This checks that new workers execute new template
"""
try:
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
self._trigger_do_action(TRIGGER_ALARM_1)
self._check_deduced(1, DEDUCED_PROPS, host_id)
except Exception as e:
self._handle_exception(e)
raise
@utils.tempest_logger
def test_evaluator_reload_with_new_template_v2(self):
"""Test reload new template e2e v2
1. raise trigger alarm
2. add the relevant template
3. delete the template.
4. check action - should be not active.
This checks that the evaluators are reloaded and run on all existing
vertices.
Checks temporary worker that was added to delete template.
"""
try:
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self._trigger_do_action(TRIGGER_ALARM_1)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
self._check_deduced(1, DEDUCED_PROPS, host_id)
v_util.delete_template(self.added_template['uuid'])
self.added_template = None
self._check_deduced(0, DEDUCED_PROPS, host_id)
except Exception as e:
self._handle_exception(e)
raise
@utils.tempest_logger
def test_evaluator_reload_with_existing_template_v2(self):
"""Test reload new template e2e v2
1.add the relevant template
2.delete the template
2.raise trigger alarm.
3. check no deduced alarm
This checks that template deleted properly and no action executed.
:return:
"""
try:
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self.added_template = v_util.add_template(TEST_TEMPLATE,
folder=FOLDER_PATH)
v_util.delete_template(self.added_template['uuid'])
self.added_template = None
self._trigger_do_action(TRIGGER_ALARM_1)
self._check_deduced(0, DEDUCED_PROPS, host_id)
except Exception as e:
self._handle_exception(e)
raise
@utils.tempest_logger
def test_evaluator_reload_with_multiple_new_template(self):
"""Test reload new template e2e
1. raise trigger alarm (template is not loaded yet)
2. add 2 new templates.
3. check both actions are executed
This checks that the evaluators are reloaded for both templates
and run on all existing vertices.
"""
try:
host_id = self.orig_host.get(VProps.VITRAGE_ID)
self._trigger_do_action(TRIGGER_ALARM_1)
self._trigger_do_action(TRIGGER_ALARM_2)
v_util.add_template(folder=FOLDER_PATH)
self.added_template = v_util.get_first_template(name=INFILE_NAME)
second_template = v_util.get_first_template(name=INFILE_NAME_2)
self._check_deduced(1, DEDUCED_PROPS, host_id)
self._check_deduced(1, DEDUCED_PROPS_2, host_id)
except Exception as e:
self._handle_exception(e)
raise
finally:
if second_template:
v_util.delete_template(second_template['uuid'])

22
vitrage_tempest_tests/tests/resources/templates/api/basic_def_template.yaml Normal file
View File

@ -0,0 +1,22 @@
metadata:
#a basic def_template file
name: basic_def_template
description: basic def_template for general tests
definitions:
entities:
- entity:
category: ALARM
type: nagios
name: host_problem
template_id: alarm
- entity:
category: RESOURCE
type: nova.host
template_id: resource
relationships:
- relationship:
source: alarm
target: resource
relationship_type: on
template_id : alarm_on_host

21
vitrage_tempest_tests/tests/resources/templates/api/basic_equivalence_templates.yaml Normal file
View File

@ -0,0 +1,21 @@
metadata:
name: entity equivalence example
equivalences:
- equivalence:
- entity:
category: ALARM
type: nagios
name: cpu_high
- entity:
category: ALARM
type: vitrage
name: cpu_high
- equivalence:
- entity:
category: ALARM
type: nagios
name: mem_low
- entity:
category: ALARM
type: vitrage
name: mem_low

66
vitrage_tempest_tests/tests/resources/templates/api/e2e_test_template/e2e_test_template_actions.yaml Normal file
View File

@ -0,0 +1,66 @@
metadata:
name: e2e_test_template_actions
description: this template includes vitrage basic actions
definitions:
entities:
- entity:
category: ALARM
name: e2e.test_template_actions.trigger.alarm1
template_id: trigger_alarm_1
- entity:
category: ALARM
type: vitrage
name: e2e.test_template_actions.deduced.alarm
template_id: deduced_alarm
- entity:
category: RESOURCE
type: nova.host
template_id: host
- entity:
category: RESOURCE
type: nova.instance
template_id: instance
relationships:
- relationship:
source: trigger_alarm_1
relationship_type: on
target: host
template_id : trigger_alarm_1_on_host
- relationship:
source: deduced_alarm
relationship_type: on
target: host
template_id : deduced_alarm_on_host
- relationship:
source: host
target: instance
relationship_type: contains
template_id: host_contains_instance
scenarios:
- scenario:
condition: trigger_alarm_1_on_host
actions:
- action:
action_type: set_state
action_target:
target: host
properties:
state: ERROR
- scenario:
condition: trigger_alarm_1_on_host
actions:
- action:
action_type: raise_alarm
action_target:
target: host
properties:
alarm_name: e2e.test_template_actions.deduced.alarm
severity: WARNING
- scenario:
condition: trigger_alarm_1_on_host and deduced_alarm_on_host
actions:
- action:
action_type: add_causal_relationship
action_target:
source: trigger_alarm_1
target: deduced_alarm

66
vitrage_tempest_tests/tests/resources/templates/api/e2e_test_template/e2e_test_template_actions_2.yaml Normal file
View File

@ -0,0 +1,66 @@
metadata:
name: e2e_test_template_actions_2
description: this template includes vitrage basic actions
definitions:
entities:
- entity:
category: ALARM
name: e2e.test_template_actions.trigger.alarm2
template_id: trigger_alarm_2
- entity:
category: ALARM
type: vitrage
name: e2e.test_template_actions.deduced.alarm2
template_id: deduced_alarm_2
- entity:
category: RESOURCE
type: nova.host
template_id: host
- entity:
category: RESOURCE
type: nova.instance
template_id: instance
relationships:
- relationship:
source: trigger_alarm_2
relationship_type: on
target: host
template_id : trigger_alarm_2_on_host
- relationship:
source: deduced_alarm_2
relationship_type: on
target: host
template_id : deduced_alarm_2_on_host
- relationship:
source: host
target: instance
relationship_type: contains
template_id: host_contains_instance
scenarios:
- scenario:
condition: trigger_alarm_2_on_host
actions:
- action:
action_type: set_state
action_target:
target: host
properties:
state: ERROR
- scenario:
condition: trigger_alarm_2_on_host
actions:
- action:
action_type: raise_alarm
action_target:
target: host
properties:
alarm_name: e2e.test_template_actions.deduced.alarm2
severity: WARNING
- scenario:
condition: trigger_alarm_2_on_host and deduced_alarm_2_on_host
actions:
- action:
action_type: add_causal_relationship
action_target:
source: trigger_alarm_2
target: deduced_alarm_2

12
vitrage_tempest_tests/tests/resources/templates/api/equivalence_template.yaml Normal file
View File

@ -0,0 +1,12 @@
metadata:
name: zabbix_host_equivalence
equivalences:
- equivalence:
- entity:
category: ALARM
type: nagios
name: host_problem
- entity:
category: ALARM
type: zabbix
name: host_problem

73
vitrage_tempest_tests/tests/resources/templates/api/host_high_memory_usage_scenarios.yaml Normal file
View File

@ -0,0 +1,73 @@
metadata:
name: host_high_memory_usage_scenarios
description: scenarios triggered by high memory usage on physical host
definitions:
entities:
- entity:
category: ALARM
type: zabbix
rawtext: Lack of available memory on server {HOST.NAME}
template_id: lack_of_available_memory_alarm
- entity:
category: ALARM
type: vitrage
name: Instance memory performance degraded
template_id: instance_memory_performance_degraded_alarm
- entity:
category: RESOURCE
type: nova.instance
template_id: instance
- entity:
category: RESOURCE
type: nova.host
template_id: host
relationships:
- relationship:
source: lack_of_available_memory_alarm
relationship_type: on
target: host
template_id : lack_of_available_memory_alarm_on_host
- relationship:
source: host
relationship_type: contains
target: instance
template_id : host_contains_instance
- relationship:
source: instance_memory_performance_degraded_alarm
relationship_type: on
target: instance
template_id : memory_performance_degraded_alarm_on_instance
scenarios:
- scenario:
condition: lack_of_available_memory_alarm_on_host and host_contains_instance
actions:
- action:
action_type: raise_alarm
action_target:
target: instance
properties:
alarm_name: Instance memory performance degraded
severity: warning
- scenario:
condition: lack_of_available_memory_alarm_on_host and host_contains_instance and memory_performance_degraded_alarm_on_instance
actions:
- action:
action_type: add_causal_relationship
action_target:
source: lack_of_available_memory_alarm
target: instance_memory_performance_degraded_alarm
- action:
action_type: set_state
action_target:
target: instance
properties:
state: SUBOPTIMAL
- scenario:
condition: lack_of_available_memory_alarm_on_host
actions:
- action:
action_type: set_state
action_target:
target: host
properties:
state: SUBOPTIMAL

2
vitrage_tempest_tests/tests/utils.py
View File

@ -167,5 +167,5 @@ def wait_for_status(max_waiting, func, **kwargs):
return True
count += 1
time.sleep(2)
LOG.info("wait_for_status - False")
LOG.error("wait_for_status - False")
return False