Added support for versioned notifications

In this changeset, I added all the required modification in order
for Watcher to enable the implementation of versioned notifications.

Change-Id: I600ecbc767583824555b016fb9fc7faf69c53b39
Partially-Implements: blueprint watcher-notifications-ovo
This commit is contained in:
Vincent Françoise 2016-10-17 17:17:56 +02:00
parent 9dc3fce3e5
commit b27e5b91b9
17 changed files with 594 additions and 31 deletions

View File

@ -0,0 +1,16 @@
{
"event_type": "infra-optim.exception",
"payload": {
"watcher_object.data": {
"exception": "NoAvailableStrategyForGoal",
"exception_message": "No strategy could be found to achieve the server_consolidation goal.",
"function_name": "_aggregate_create_in_db",
"module_name": "watcher.objects.aggregate"
},
"watcher_object.name": "ExceptionPayload",
"watcher_object.namespace": "watcher",
"watcher_object.version": "1.0"
},
"priority": "ERROR",
"publisher_id": "watcher-api:fake-mini"
}

View File

@ -39,8 +39,7 @@ class ApplierAPI(service.Service):
raise exception.InvalidUuidOrName(name=action_plan_uuid)
return self.conductor_client.call(
context.to_dict(), 'launch_action_plan',
action_plan_uuid=action_plan_uuid)
context, 'launch_action_plan', action_plan_uuid=action_plan_uuid)
class ApplierAPIManager(object):

View File

@ -35,8 +35,7 @@ def main():
host, port = cfg.CONF.api.host, cfg.CONF.api.port
protocol = "http" if not CONF.api.enable_ssl_api else "https"
# Build and start the WSGI app
server = service.WSGIService(
'watcher-api', CONF.api.enable_ssl_api)
server = service.WSGIService('watcher-api', CONF.api.enable_ssl_api)
if host == '127.0.0.1':
LOG.info(_LI('serving on 127.0.0.1:%(port)s, '

View File

@ -121,7 +121,7 @@ class RequestContextSerializer(messaging.Serializer):
return self._base.deserialize_entity(context, entity)
def serialize_context(self, context):
return context
return context.to_dict()
def deserialize_context(self, context):
return watcher_context.RequestContext.from_dict(context)
@ -146,8 +146,6 @@ def get_server(target, endpoints, serializer=None):
serializer=serializer)
def get_notifier(service=None, host=None, publisher_id=None):
def get_notifier(publisher_id):
assert NOTIFIER is not None
if not publisher_id:
publisher_id = "%s.%s" % (service, host or CONF.host)
return NOTIFIER.prepare(publisher_id=publisher_id)

View File

@ -74,21 +74,21 @@ Singleton = service.Singleton
class WSGIService(service.ServiceBase):
"""Provides ability to launch Watcher API from wsgi app."""
def __init__(self, name, use_ssl=False):
def __init__(self, service_name, use_ssl=False):
"""Initialize, but do not start the WSGI server.
:param name: The name of the WSGI server given to the loader.
:param service_name: The service name of the WSGI server.
:param use_ssl: Wraps the socket in an SSL context if True.
"""
self.name = name
self.service_name = service_name
self.app = app.VersionSelectorApplication()
self.workers = (CONF.api.workers or
processutils.get_worker_count())
self.server = wsgi.Server(CONF, name, self.app,
self.server = wsgi.Server(CONF, self.service_name, self.app,
host=CONF.api.host,
port=CONF.api.port,
use_ssl=use_ssl,
logger_name=name)
logger_name=self.service_name)
def start(self):
"""Start serving this service using loaded configuration"""
@ -307,7 +307,7 @@ class Service(service.ServiceBase, dispatcher.EventDispatcher):
def check_api_version(self, context):
api_manager_version = self.conductor_client.call(
context.to_dict(), 'check_api_version',
context, 'check_api_version',
api_version=self.api_version)
return api_manager_version

View File

@ -19,8 +19,6 @@
import abc
import six
from watcher.common import rpc
@six.add_metaclass(abc.ABCMeta)
class NotificationEndpoint(object):
@ -38,10 +36,3 @@ class NotificationEndpoint(object):
@property
def cluster_data_model(self):
return self.collector.cluster_data_model
@property
def notifier(self):
if self._notifier is None:
self._notifier = rpc.get_notifier('decision-engine')
return self._notifier

View File

@ -43,7 +43,7 @@ class DecisionEngineAPI(service.Service):
raise exception.InvalidUuidOrName(name=audit_uuid)
return self.conductor_client.call(
context.to_dict(), 'trigger_audit', audit_uuid=audit_uuid)
context, 'trigger_audit', audit_uuid=audit_uuid)
class DecisionEngineAPIManager(object):

View File

@ -136,6 +136,7 @@ class WatcherPersistentObject(object):
:param db_object: A DB model of the object
:param eager: Enable the loading of object fields (Default: False)
:return: The object of the class with the database entity added
"""
obj_class = type(obj)
object_fields = obj_class.object_fields

View File

@ -24,8 +24,10 @@ from oslo_versionedobjects import fields
LOG = log.getLogger(__name__)
BaseEnumField = fields.BaseEnumField
BooleanField = fields.BooleanField
DateTimeField = fields.DateTimeField
Enum = fields.Enum
IntegerField = fields.IntegerField
ListOfStringsField = fields.ListOfStringsField
ObjectField = fields.ObjectField
@ -88,3 +90,53 @@ class FlexibleListOfDictField(fields.AutoTypedField):
if self.nullable:
return []
super(FlexibleListOfDictField, self)._null(obj, attr)
# ### Notification fields ### #
class BaseWatcherEnum(Enum):
ALL = ()
def __init__(self, **kwargs):
super(BaseWatcherEnum, self).__init__(valid_values=self.__class__.ALL)
class NotificationPriority(BaseWatcherEnum):
CRITICAL = 'critical'
DEBUG = 'debug'
INFO = 'info'
ERROR = 'error'
SAMPLE = 'sample'
WARNING = 'warn'
ALL = (CRITICAL, DEBUG, INFO, ERROR, SAMPLE, WARNING)
class NotificationPhase(BaseWatcherEnum):
START = 'start'
END = 'end'
ERROR = 'error'
ALL = (START, END, ERROR)
class NotificationAction(BaseWatcherEnum):
CREATE = 'create'
UPDATE = 'update'
EXCEPTION = 'exception'
DELETE = 'delete'
ALL = (CREATE, UPDATE, EXCEPTION, DELETE)
class NotificationPriorityField(BaseEnumField):
AUTO_TYPE = NotificationPriority()
class NotificationPhaseField(BaseEnumField):
AUTO_TYPE = NotificationPhase()
class NotificationActionField(BaseEnumField):
AUTO_TYPE = NotificationAction()

View File

@ -0,0 +1,166 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from watcher.common import rpc
from watcher.objects import base
from watcher.objects import fields as wfields
@base.WatcherObjectRegistry.register_if(False)
class NotificationObject(base.WatcherObject):
"""Base class for every notification related versioned object."""
# Version 1.0: Initial version
VERSION = '1.0'
def __init__(self, **kwargs):
super(NotificationObject, self).__init__(**kwargs)
# The notification objects are created on the fly when watcher emits
# the notification. This causes that every object shows every field as
# changed. We don't want to send this meaningless information so we
# reset the object after creation.
self.obj_reset_changes(recursive=False)
@base.WatcherObjectRegistry.register_notification
class EventType(NotificationObject):
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'object': wfields.StringField(),
'action': wfields.NotificationActionField(),
'phase': wfields.NotificationPhaseField(nullable=True),
}
def to_notification_event_type_field(self):
"""Serialize the object to the wire format."""
s = '%s.%s' % (self.object, self.action)
if self.obj_attr_is_set('phase'):
s += '.%s' % self.phase
return s
@base.WatcherObjectRegistry.register_if(False)
class NotificationPayloadBase(NotificationObject):
"""Base class for the payload of versioned notifications."""
# SCHEMA defines how to populate the payload fields. It is a dictionary
# where every key value pair has the following format:
# <payload_field_name>: (<data_source_name>,
# <field_of_the_data_source>)
# The <payload_field_name> is the name where the data will be stored in the
# payload object, this field has to be defined as a field of the payload.
# The <data_source_name> shall refer to name of the parameter passed as
# kwarg to the payload's populate_schema() call and this object will be
# used as the source of the data. The <field_of_the_data_source> shall be
# a valid field of the passed argument.
# The SCHEMA needs to be applied with the populate_schema() call before the
# notification can be emitted.
# The value of the payload.<payload_field_name> field will be set by the
# <data_source_name>.<field_of_the_data_source> field. The
# <data_source_name> will not be part of the payload object internal or
# external representation.
# Payload fields that are not set by the SCHEMA can be filled in the same
# way as in any versioned object.
SCHEMA = {}
# Version 1.0: Initial version
VERSION = '1.0'
def __init__(self, **kwargs):
super(NotificationPayloadBase, self).__init__(**kwargs)
self.populated = not self.SCHEMA
def populate_schema(self, **kwargs):
"""Populate the object based on the SCHEMA and the source objects
:param kwargs: A dict contains the source object at the key defined in
the SCHEMA
"""
for key, (obj, field) in self.SCHEMA.items():
source = kwargs[obj]
if source.obj_attr_is_set(field):
setattr(self, key, getattr(source, field))
self.populated = True
# the schema population will create changed fields but we don't need
# this information in the notification
self.obj_reset_changes(recursive=False)
@base.WatcherObjectRegistry.register_notification
class NotificationPublisher(NotificationObject):
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'host': wfields.StringField(nullable=False),
'binary': wfields.StringField(nullable=False),
}
@base.WatcherObjectRegistry.register_if(False)
class NotificationBase(NotificationObject):
"""Base class for versioned notifications.
Every subclass shall define a 'payload' field.
"""
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'priority': wfields.NotificationPriorityField(),
'event_type': wfields.ObjectField('EventType'),
'publisher': wfields.ObjectField('NotificationPublisher'),
}
def _emit(self, context, event_type, publisher_id, payload):
notifier = rpc.get_notifier(publisher_id)
notify = getattr(notifier, self.priority)
notify(context, event_type=event_type, payload=payload)
def emit(self, context):
"""Send the notification."""
assert self.payload.populated
# Note(gibi): notification payload will be a newly populated object
# therefore every field of it will look changed so this does not carry
# any extra information so we drop this from the payload.
self.payload.obj_reset_changes(recursive=False)
self._emit(
context,
event_type=self.event_type.to_notification_event_type_field(),
publisher_id='%s:%s' % (self.publisher.binary,
self.publisher.host),
payload=self.payload.obj_to_primitive())
def notification_sample(sample):
"""Provide a notification sample of the decatorated notification.
Class decorator to attach the notification sample information
to the notification object for documentation generation purposes.
:param sample: the path of the sample json file relative to the
doc/notification_samples/ directory in the watcher
repository root.
"""
def wrap(cls):
if not getattr(cls, 'samples', None):
cls.samples = [sample]
else:
cls.samples.append(sample)
return cls
return wrap

View File

@ -0,0 +1,52 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import inspect
import six
from watcher.objects import base as base
from watcher.objects import fields as wfields
from watcher.objects.notifications import base as notificationbase
@base.WatcherObjectRegistry.register_notification
class ExceptionPayload(notificationbase.NotificationPayloadBase):
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'module_name': wfields.StringField(),
'function_name': wfields.StringField(),
'exception': wfields.StringField(),
'exception_message': wfields.StringField()
}
@classmethod
def from_exception(cls, fault):
trace = inspect.trace()[-1]
# TODO(gibi): apply strutils.mask_password on exception_message and
# consider emitting the exception_message only if the safe flag is
# true in the exception like in the REST API
return cls(
function_name=trace[3],
module_name=inspect.getmodule(trace[0]).__name__,
exception=fault.__class__.__name__,
exception_message=six.text_type(fault))
@base.WatcherObjectRegistry.register_notification
class ExceptionNotification(notificationbase.NotificationBase):
# Version 1.0: Initial version
VERSION = '1.0'
fields = {
'payload': wfields.ObjectField('ExceptionPayload')
}

View File

@ -41,7 +41,7 @@ class TestApplierAPI(base.TestCase):
expected_context = self.context
self.api.check_api_version(expected_context)
mock_call.assert_called_once_with(
expected_context.to_dict(),
expected_context,
'check_api_version',
api_version=rpcapi.ApplierAPI().API_VERSION)
@ -50,7 +50,7 @@ class TestApplierAPI(base.TestCase):
action_plan_uuid = utils.generate_uuid()
self.api.launch_action_plan(self.context, action_plan_uuid)
mock_call.assert_called_once_with(
self.context.to_dict(),
self.context,
'launch_action_plan',
action_plan_uuid=action_plan_uuid)

View File

@ -38,8 +38,7 @@ class TestDecisionEngineAPI(base.TestCase):
expected_context = self.context
self.api.check_api_version(expected_context)
mock_call.assert_called_once_with(
expected_context.to_dict(),
'check_api_version',
expected_context, 'check_api_version',
api_version=rpcapi.DecisionEngineAPI().api_version)
def test_execute_audit_throw_exception(self):
@ -52,6 +51,5 @@ class TestDecisionEngineAPI(base.TestCase):
with mock.patch.object(om.RPCClient, 'call') as mock_call:
audit_uuid = utils.generate_uuid()
self.api.trigger_audit(self.context, audit_uuid)
mock_call.assert_called_once_with(self.context.to_dict(),
'trigger_audit',
audit_uuid=audit_uuid)
mock_call.assert_called_once_with(
self.context, 'trigger_audit', audit_uuid=audit_uuid)

View File

@ -0,0 +1,272 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import collections
import mock
from oslo_versionedobjects import fixture
from watcher.common import rpc
from watcher.objects import base
from watcher.objects import fields as wfields
from watcher.objects.notifications import base as notificationbase
from watcher.tests import base as testbase
from watcher.tests.objects import test_objects
class TestNotificationBase(testbase.TestCase):
@base.WatcherObjectRegistry.register_if(False)
class TestObject(base.WatcherObject):
VERSION = '1.0'
fields = {
'field_1': wfields.StringField(),
'field_2': wfields.IntegerField(),
'not_important_field': wfields.IntegerField(),
}
@base.WatcherObjectRegistry.register_if(False)
class TestNotificationPayload(notificationbase.NotificationPayloadBase):
VERSION = '1.0'
SCHEMA = {
'field_1': ('source_field', 'field_1'),
'field_2': ('source_field', 'field_2'),
}
fields = {
'extra_field': wfields.StringField(), # filled by ctor
'field_1': wfields.StringField(), # filled by the schema
'field_2': wfields.IntegerField(), # filled by the schema
}
def populate_schema(self, source_field):
super(TestNotificationBase.TestNotificationPayload,
self).populate_schema(source_field=source_field)
@base.WatcherObjectRegistry.register_if(False)
class TestNotificationPayloadEmptySchema(
notificationbase.NotificationPayloadBase):
VERSION = '1.0'
fields = {
'extra_field': wfields.StringField(), # filled by ctor
}
@notificationbase.notification_sample('test-update-1.json')
@notificationbase.notification_sample('test-update-2.json')
@base.WatcherObjectRegistry.register_if(False)
class TestNotification(notificationbase.NotificationBase):
VERSION = '1.0'
fields = {
'payload': wfields.ObjectField('TestNotificationPayload')
}
@base.WatcherObjectRegistry.register_if(False)
class TestNotificationEmptySchema(notificationbase.NotificationBase):
VERSION = '1.0'
fields = {
'payload': wfields.ObjectField(
'TestNotificationPayloadEmptySchema')
}
expected_payload = {
'watcher_object.name': 'TestNotificationPayload',
'watcher_object.data': {
'extra_field': 'test string',
'field_1': 'test1',
'field_2': 42},
'watcher_object.version': '1.0',
'watcher_object.namespace': 'watcher'}
def setUp(self):
super(TestNotificationBase, self).setUp()
self.my_obj = self.TestObject(field_1='test1',
field_2=42,
not_important_field=13)
self.payload = self.TestNotificationPayload(
extra_field='test string')
self.payload.populate_schema(source_field=self.my_obj)
self.notification = self.TestNotification(
event_type=notificationbase.EventType(
object='test_object',
action=wfields.NotificationAction.UPDATE,
phase=wfields.NotificationPhase.START),
publisher=notificationbase.NotificationPublisher(
host='fake-host', binary='watcher-fake'),
priority=wfields.NotificationPriority.INFO,
payload=self.payload)
def _verify_notification(self, mock_notifier, mock_context,
expected_event_type,
expected_payload):
mock_notifier.prepare.assert_called_once_with(
publisher_id='watcher-fake:fake-host')
mock_notify = mock_notifier.prepare.return_value.info
self.assertTrue(mock_notify.called)
self.assertEqual(mock_notify.call_args[0][0], mock_context)
self.assertEqual(mock_notify.call_args[1]['event_type'],
expected_event_type)
actual_payload = mock_notify.call_args[1]['payload']
self.assertEqual(expected_payload, actual_payload)
@mock.patch.object(rpc, 'NOTIFIER')
def test_emit_notification(self, mock_notifier):
mock_context = mock.Mock()
mock_context.to_dict.return_value = {}
self.notification.emit(mock_context)
self._verify_notification(
mock_notifier,
mock_context,
expected_event_type='test_object.update.start',
expected_payload=self.expected_payload)
@mock.patch.object(rpc, 'NOTIFIER')
def test_emit_event_type_without_phase(self, mock_notifier):
noti = self.TestNotification(
event_type=notificationbase.EventType(
object='test_object',
action=wfields.NotificationAction.UPDATE),
publisher=notificationbase.NotificationPublisher(
host='fake-host', binary='watcher-fake'),
priority=wfields.NotificationPriority.INFO,
payload=self.payload)
mock_context = mock.Mock()
mock_context.to_dict.return_value = {}
noti.emit(mock_context)
self._verify_notification(
mock_notifier,
mock_context,
expected_event_type='test_object.update',
expected_payload=self.expected_payload)
@mock.patch.object(rpc, 'NOTIFIER')
def test_not_possible_to_emit_if_not_populated(self, mock_notifier):
non_populated_payload = self.TestNotificationPayload(
extra_field='test string')
noti = self.TestNotification(
event_type=notificationbase.EventType(
object='test_object',
action=wfields.NotificationAction.UPDATE),
publisher=notificationbase.NotificationPublisher(
host='fake-host', binary='watcher-fake'),
priority=wfields.NotificationPriority.INFO,
payload=non_populated_payload)
mock_context = mock.Mock()
self.assertRaises(AssertionError, noti.emit, mock_context)
self.assertFalse(mock_notifier.called)
@mock.patch.object(rpc, 'NOTIFIER')
def test_empty_schema(self, mock_notifier):
non_populated_payload = self.TestNotificationPayloadEmptySchema(
extra_field='test string')
noti = self.TestNotificationEmptySchema(
event_type=notificationbase.EventType(
object='test_object',
action=wfields.NotificationAction.UPDATE),
publisher=notificationbase.NotificationPublisher(
host='fake-host', binary='watcher-fake'),
priority=wfields.NotificationPriority.INFO,
payload=non_populated_payload)
mock_context = mock.Mock()
mock_context.to_dict.return_value = {}
noti.emit(mock_context)
self._verify_notification(
mock_notifier,
mock_context,
expected_event_type='test_object.update',
expected_payload={
'watcher_object.name': 'TestNotificationPayloadEmptySchema',
'watcher_object.data': {'extra_field': 'test string'},
'watcher_object.version': '1.0',
'watcher_object.namespace': 'watcher'})
def test_sample_decorator(self):
self.assertEqual(2, len(self.TestNotification.samples))
self.assertIn('test-update-1.json', self.TestNotification.samples)
self.assertIn('test-update-2.json', self.TestNotification.samples)
expected_notification_fingerprints = {
'EventType': '1.0-92100a9f0908da98dfcfff9c42e0018c',
'NotificationPublisher': '1.0-bbbc1402fb0e443a3eb227cc52b61545',
}
class TestNotificationObjectVersions(testbase.TestCase):
def setUp(self):
super(TestNotificationObjectVersions, self).setUp()
base.WatcherObjectRegistry.register_notification_objects()
def test_versions(self):
checker = fixture.ObjectVersionChecker(
test_objects.get_watcher_objects())
expected_notification_fingerprints.update(
test_objects.expected_object_fingerprints)
expected, actual = checker.test_hashes(
expected_notification_fingerprints)
self.assertEqual(expected, actual,
'Some notification objects have changed; please make '
'sure the versions have been bumped, and then update '
'their hashes here.')
def test_notification_payload_version_depends_on_the_schema(self):
@base.WatcherObjectRegistry.register_if(False)
class TestNotificationPayload(
notificationbase.NotificationPayloadBase):
VERSION = '1.0'
SCHEMA = {
'field_1': ('source_field', 'field_1'),
'field_2': ('source_field', 'field_2'),
}
fields = {
'extra_field': wfields.StringField(), # filled by ctor
'field_1': wfields.StringField(), # filled by the schema
'field_2': wfields.IntegerField(), # filled by the schema
}
checker = fixture.ObjectVersionChecker(
{'TestNotificationPayload': (TestNotificationPayload,)})
old_hash = checker.get_hashes(extra_data_func=get_extra_data)
TestNotificationPayload.SCHEMA['field_3'] = ('source_field',
'field_3')
new_hash = checker.get_hashes(extra_data_func=get_extra_data)
self.assertNotEqual(old_hash, new_hash)
def get_extra_data(obj_class):
extra_data = tuple()
# Get the SCHEMA items to add to the fingerprint
# if we are looking at a notification
if issubclass(obj_class, notificationbase.NotificationPayloadBase):
schema_data = collections.OrderedDict(
sorted(obj_class.SCHEMA.items()))
extra_data += (schema_data,)
return extra_data

View File

@ -422,6 +422,25 @@ expected_object_fingerprints = {
}
def get_watcher_objects():
"""Get Watcher versioned objects
This returns a dict of versioned objects which are
in the Watcher project namespace only. ie excludes
objects from os-vif and other 3rd party modules
:return: a dict mapping class names to lists of versioned objects
"""
all_classes = base.WatcherObjectRegistry.obj_classes()
watcher_classes = {}
for name in all_classes:
objclasses = all_classes[name]
if (objclasses[0].OBJ_PROJECT_NAMESPACE !=
base.WatcherObject.OBJ_PROJECT_NAMESPACE):
continue
watcher_classes[name] = objclasses
return watcher_classes
class TestObjectVersions(test_base.TestCase):
def test_object_version_check(self):