From 3d41d2e8e537b3679832393eda5b31df28a9bc63 Mon Sep 17 00:00:00 2001 From: Paul Michali Date: Fri, 4 Dec 2015 19:32:45 +0000 Subject: [PATCH] Adding callback mechanism Adding the callback mechanism to neutron-lib. Added code, test, and devref. Added additional test coverage. Added missing __init__.py files so tests are run. Needed oslo.log added to requirements. Once this is upstreamed, we can modify the callback mechanism. Change-Id: Ib16f3942e8ac2ddbfc8ff6919863ec9ad197e5b6 Implements: blueprint neutron-lib --- doc/source/devref/callbacks.rst | 420 ++++++++++++++++++ doc/source/devref/index.rst | 4 +- neutron_lib/_callbacks/__init__.py | 0 neutron_lib/_callbacks/events.py | 30 ++ neutron_lib/_callbacks/exceptions.py | 40 ++ neutron_lib/_callbacks/manager.py | 162 +++++++ neutron_lib/_callbacks/registry.py | 48 ++ neutron_lib/_callbacks/resources.py | 22 + neutron_lib/tests/unit/__init__.py | 0 neutron_lib/tests/unit/callbacks/__init__.py | 0 .../callbacks/test_callback_exceptions.py | 56 +++ .../tests/unit/callbacks/test_manager.py | 218 +++++++++ .../tests/unit/callbacks/test_registry.py | 59 +++ requirements.txt | 1 + tox.ini | 4 +- 15 files changed, 1060 insertions(+), 4 deletions(-) create mode 100644 doc/source/devref/callbacks.rst create mode 100644 neutron_lib/_callbacks/__init__.py create mode 100644 neutron_lib/_callbacks/events.py create mode 100644 neutron_lib/_callbacks/exceptions.py create mode 100644 neutron_lib/_callbacks/manager.py create mode 100644 neutron_lib/_callbacks/registry.py create mode 100644 neutron_lib/_callbacks/resources.py create mode 100644 neutron_lib/tests/unit/__init__.py create mode 100644 neutron_lib/tests/unit/callbacks/__init__.py create mode 100644 neutron_lib/tests/unit/callbacks/test_callback_exceptions.py create mode 100644 neutron_lib/tests/unit/callbacks/test_manager.py create mode 100644 neutron_lib/tests/unit/callbacks/test_registry.py diff --git a/doc/source/devref/callbacks.rst b/doc/source/devref/callbacks.rst new file mode 100644 index 000000000..3ddbca4ca --- /dev/null +++ b/doc/source/devref/callbacks.rst @@ -0,0 +1,420 @@ +.. + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + + Convention for heading levels in Neutron devref: + ======= Heading 0 (reserved for the title in a document) + ------- Heading 1 + ~~~~~~~ Heading 2 + +++++++ Heading 3 + ''''''' Heading 4 + (Avoid deeper levels because they do not render well.) + + +Neutron Callback System +======================= + +In Neutron, core and service components may need to cooperate during the +execution of certain operations, or they may need to react upon the occurrence +of certain events. For instance, when a Neutron resource is associated to +multiple services, the components in charge of these services may need to play +an active role in determining what the right state of the resource needs to be. + +The cooperation may be achieved by making each object aware of each other, but +this leads to tight coupling, or alternatively it can be achieved by using a +callback-based system, where the same objects are allowed to cooperate in a +loose manner. + +This is particularly important since the spin off of the advanced services like +VPN, Firewall and Load Balancer, where each service's codebase lives independently +from the core and from one another. This means that the tight coupling is no longer +a practical solution for object cooperation. In addition to this, if more services +are developed independently, there is no viable integration between them and the +Neutron core. A callback system, and its registry, tries to address these issues. + +In object-oriented software systems, method invocation is also known as message +passing: an object passes a message to another object, and it may or may not expect +a message back. This point-to-point interaction can take place between the parties +directly involved in the communication, or it can happen via an intermediary. The +intermediary is then in charge of keeping track of who is interested in the messages +and in delivering the messages forth and back, when required. As mentioned earlier, +the use of an intermediary has the benefit of decoupling the parties involved +in the communications, as now they only need to know about the intermediary; the +other benefit is that the use of an intermediary opens up the possibility of +multiple party communication: more than one object can express interest in +receiving the same message, and the same message can be delivered to more than +one object. To this aim, the intermediary is the entity that exists throughout +the system lifecycle, as it needs to be able to track whose interest is associated +to what message. + +In a design for a system that enables callback-based communication, the following +aspects need to be taken into account: + +* how to become consumer of messages (i.e. how to be on the receiving end of the message); +* how to become producer of messages (i.e. how to be on the sending end of the message); +* how to consume/produce messages selectively; + +Translate and narrow this down to Neutron needs, and this means the design of a callback +system where messages are about lifecycle events (e.g. before creation, before +deletion, etc.) of Neutron resources (e.g. networks, routers, ports, etc.), where the +various parties can express interest in knowing when these events for a specific +resources take place. + +Rather than keeping the conversation abstract, let us delve into some examples, that would +help understand better some of the principles behind the provided mechanism. + + +Subscribing to events +--------------------- + +Imagine that you have entity A, B, and C that have some common business over router creation. +A wants to tell B and C that the router has been created and that they need to get on and +do whatever they are supposed to do. In a callback-less world this would work like so: + +:: + + # A is done creating the resource + # A gets hold of the references of B and C + # A calls B + # A calls C + B->my_random_method_for_knowing_about_router_created() + C->my_random_very_difficult_to_remember_method_about_router_created() + +If B and/or C change, things become sour. In a callback-based world, things become a lot +more uniform and straightforward: + +:: + + # B and C ask I to be notified when A is done creating the resource + + # ... + # A is done creating the resource + # A gets hold of the reference to the intermediary I + # A calls I + I->notify() + +Since B and C will have expressed interest in knowing about A's business, 'I' will +deliver the messages to B and C. If B and C changes, A and 'I' do not need to change. + +In practical terms this scenario would be translated in the code below: + +:: + + from neutron_lib.callbacks import events + from neutron_lib.callbacks import resources + from neutron_lib.callbacks import registry + + + def callback1(resource, event, trigger, **kwargs): + print('Callback1 called by trigger: ', trigger) + print('kwargs: ', kwargs) + + def callback2(resource, event, trigger, **kwargs): + print('Callback2 called by trigger: ', trigger) + print('kwargs: ', kwargs) + + + # B and C express interest with I + registry.subscribe(callback1, resources.ROUTER, events.BEFORE_CREATE) + registry.subscribe(callback2, resources.ROUTER, events.BEFORE_CREATE) + print('Subscribed') + + + # A notifies + def do_notify(): + kwargs = {'foo': 'bar'} + registry.notify(resources.ROUTER, events.BEFORE_CREATE, do_notify, **kwargs) + + + print('Notifying...') + do_notify() + + +The output is: + +:: + + > Subscribed + > Notifying... + > Callback2 called by trigger: + > kwargs: {'foo': 'bar'} + > Callback1 called by trigger: + > kwargs: {'foo': 'bar'} + +Thanks to the intermediary existence throughout the life of the system, A, B, and C +are flexible to evolve their internals, dynamics, and lifecycles. + + +Subscribing and aborting events +------------------------------- + +Interestingly in Neutron, certain events may need to be forbidden from happening due to the +nature of the resources involved. To this aim, the callback-based mechanism has been designed +to support a use case where, when callbacks subscribe to specific events, the action that +results from it, may lead to the propagation of a message back to the sender, so that it itself +can be alerted and stop the execution of the activity that led to the message dispatch in the +first place. + +The typical example is where a resource, like a router, is used by one or more high-level +service(s), like a VPN or a Firewall, and actions like interface removal or router destruction +cannot not take place, because the resource is shared. + +To address this scenario, special events are introduced, 'BEFORE_*' events, to which callbacks +can subscribe and have the opportunity to 'abort', by raising an exception when notified. + +Since multiple callbacks may express an interest in the same event for a particular resource, +and since callbacks are executed independently from one another, this may lead to situations +where notifications that occurred before the exception must be aborted. To this aim, when an +exception occurs during the notification process, an abort_* event is propagated immediately +after. It is up to the callback developer to determine whether subscribing to an abort +notification is required in order to revert the actions performed during the initial execution +of the callback (when the BEFORE_* event was fired). Exceptions caused by callbacks registered +to abort events are ignored. The snippet below shows this in action: + +:: + + from neutron_lib.callbacks import events + from neutron_lib.callbacks import exceptions + from neutron_lib.callbacks import resources + from neutron_lib.callbacks import registry + + + def callback1(resource, event, trigger, **kwargs): + raise Exception('I am failing!') + + def callback2(resource, event, trigger, **kwargs): + print('Callback2 called by %s on event %s' % (trigger, event)) + + + registry.subscribe(callback1, resources.ROUTER, events.BEFORE_CREATE) + registry.subscribe(callback2, resources.ROUTER, events.BEFORE_CREATE) + registry.subscribe(callback2, resources.ROUTER, events.ABORT_CREATE) + print('Subscribed') + + + def do_notify(): + kwargs = {'foo': 'bar'} + registry.notify(resources.ROUTER, events.BEFORE_CREATE, do_notify, **kwargs) + + + print('Notifying...') + try: + do_notify() + except exceptions.CallbackFailure as e: + print('Error: ', e) + +The output is: + +:: + + > Subscribed + > Notifying... + > Callback2 called by on event before_create + > Callback2 called by on event abort_create + > Error: Callback __main__.callback1 failed with "I am failing!" + +In this case, upon the notification of the BEFORE_CREATE event, Callback1 triggers an exception +that can be used to stop the action from taking place in do_notify(). On the other end, Callback2 +will be executing twice, once for dealing with the BEFORE_CREATE event, and once to undo the +actions during the ABORT_CREATE event. It is worth noting that it is not mandatory to have +the same callback register to both BEFORE_* and the respective ABORT_* event; as a matter of +fact, it is best to make use of different callbacks to keep the two logic separate. + + +Unsubscribing to events +----------------------- + +There are a few options to unsubscribe registered callbacks: + +* clear(): it unsubscribes all subscribed callbacks: this can be useful especially when + winding down the system, and notifications shall no longer be triggered. +* unsubscribe(): it selectively unsubscribes a callback for a specific resource's event. + Say callback C has subscribed to event A for resource R, any notification of event A + for resource R will no longer be handed over to C, after the unsubscribe() invocation. +* unsubscribe_by_resource(): say that callback C has subscribed to event A, B, and C for + resource R, any notification of events related to resource R will no longer be handed + over to C, after the unsubscribe_by_resource() invocation. +* unsubscribe_all(): say that callback C has subscribed to events A, B for resource R1, + and events C, D for resource R2, any notification of events pertaining resources R1 and + R2 will no longer be handed over to C, after the unsubscribe_all() invocation. + +The snippet below shows these concepts in action: + +:: + + from neutron_lib.callbacks import events + from neutron_lib.callbacks import exceptions + from neutron_lib.callbacks import resources + from neutron_lib.callbacks import registry + + + def callback1(resource, event, trigger, **kwargs): + print('Callback1 called by %s on event %s for resource %s' % (trigger, event, resource)) + + + def callback2(resource, event, trigger, **kwargs): + print('Callback2 called by %s on event %s for resource %s' % (trigger, event, resource)) + + + registry.subscribe(callback1, resources.ROUTER, events.BEFORE_READ) + registry.subscribe(callback1, resources.ROUTER, events.BEFORE_CREATE) + registry.subscribe(callback1, resources.ROUTER, events.AFTER_DELETE) + registry.subscribe(callback1, resources.PORT, events.BEFORE_UPDATE) + registry.subscribe(callback2, resources.ROUTER_GATEWAY, events.BEFORE_UPDATE) + print('Subscribed') + + + def do_notify(): + print('Notifying...') + kwargs = {'foo': 'bar'} + registry.notify(resources.ROUTER, events.BEFORE_READ, do_notify, **kwargs) + registry.notify(resources.ROUTER, events.BEFORE_CREATE, do_notify, **kwargs) + registry.notify(resources.ROUTER, events.AFTER_DELETE, do_notify, **kwargs) + registry.notify(resources.PORT, events.BEFORE_UPDATE, do_notify, **kwargs) + registry.notify(resources.ROUTER_GATEWAY, events.BEFORE_UPDATE, do_notify, **kwargs) + + + do_notify() + registry.unsubscribe(callback1, resources.ROUTER, events.BEFORE_READ) + do_notify() + registry.unsubscribe_by_resource(callback1, resources.PORT) + do_notify() + registry.unsubscribe_all(callback1) + do_notify() + registry.clear() + do_notify() + +The output is: + +:: + + Subscribed + Notifying... + Callback1 called by on event before_read for resource router + Callback1 called by on event before_create for resource router + Callback1 called by on event after_delete for resource router + Callback1 called by on event before_update for resource port + Callback2 called by on event before_update for resource router_gateway + Notifying... + Callback1 called by on event before_create for resource router + Callback1 called by on event after_delete for resource router + Callback1 called by on event before_update for resource port + Callback2 called by on event before_update for resource router_gateway + Notifying... + Callback1 called by on event before_create for resource router + Callback1 called by on event after_delete for resource router + Callback2 called by on event before_update for resource router_gateway + Notifying... + Callback2 called by on event before_update for resource router_gateway + Notifying... + + +FAQ +--- + +Can I use the callbacks registry to subscribe and notify non-core resources and events? + + Short answer is yes. The callbacks module defines literals for what are considered core Neutron + resources and events. However, the ability to subscribe/notify is not limited to these as you + can use your own defined resources and/or events. Just make sure you use string literals, as + typos are common, and the registry does not provide any runtime validation. Therefore, make + sure you test your code! + +What is the relationship between Callbacks and Taskflow? + + There is no overlap between Callbacks and Taskflow or mutual exclusion; as matter of fact they + can be combined; You could have a callback that goes on and trigger a taskflow. It is a nice + way of separating implementation from abstraction, because you can keep the callback in place + and change Taskflow with something else. + +Is there any ordering guarantee during notifications? + + No, the ordering in which callbacks are notified is completely arbitrary by design: callbacks + should know nothing about each other, and ordering should not matter; a callback will always be + notified and its outcome should always be the same regardless as to in which order is it + notified. Priorities can be a future extension, if a use case arises that require enforced + ordering. + +How is the the notifying object expected to interact with the subscribing objects? + + The ``notify`` method implements a one-way communication paradigm: the notifier sends a message + without expecting a response back (in other words it fires and forget). However, due to the nature + of Python, the payload can be mutated by the subscribing objects, and this can lead to unexpected + behavior of your code, if you assume that this is the intentional design. Bear in mind, that + passing-by-value using deepcopy was not chosen for efficiency reasons. Having said that, if you + intend for the notifier object to expect a response, then the notifier itself would need to act + as a subscriber. + +Is the registry thread-safe? + + Short answer is no: it is not safe to make mutations while callbacks are being called (more + details as to why can be found `here `_). + A mutation could happen if a 'subscribe'/'unsubscribe' operation interleaves with the execution + of the notify loop. Albeit there is a possibility that things may end up in a bad state, the + registry works correctly under the assumption that subscriptions happen at the very beginning + of the life of the process and that the unsubscriptions (if any) take place at the very end. + In this case, chances that things do go badly may be pretty slim. Making the registry + thread-safe will be considered as a future improvement. + +What kind of function can be a callback? + + Anything you fancy: lambdas, 'closures', class, object or module methods. For instance: + +:: + + from neutron_lib.callbacks import events + from neutron_lib.callbacks import resources + from neutron_lib.callbacks import registry + + + def callback1(resource, event, trigger, **kwargs): + print('module callback') + + + class MyCallback(object): + + def callback2(self, resource, event, trigger, **kwargs): + print('object callback') + + @classmethod + def callback3(cls, resource, event, trigger, **kwargs): + print('class callback') + + + c = MyCallback() + registry.subscribe(callback1, resources.ROUTER, events.BEFORE_CREATE) + registry.subscribe(c.callback2, resources.ROUTER, events.BEFORE_CREATE) + registry.subscribe(MyCallback.callback3, resources.ROUTER, events.BEFORE_CREATE) + + def do_notify(): + def nested_subscribe(resource, event, trigger, **kwargs): + print('nested callback') + + registry.subscribe(nested_subscribe, resources.ROUTER, events.BEFORE_CREATE) + + kwargs = {'foo': 'bar'} + registry.notify(resources.ROUTER, events.BEFORE_CREATE, do_notify, **kwargs) + + + print('Notifying...') + do_notify() + +And the output is going to be: + +:: + + Notifying... + module callback + object callback + class callback + nested callback diff --git a/doc/source/devref/index.rst b/doc/source/devref/index.rst index a5044fff2..835ec344c 100644 --- a/doc/source/devref/index.rst +++ b/doc/source/devref/index.rst @@ -32,9 +32,7 @@ Neutron Lib Internals .. toctree:: :maxdepth: 3 -.. todo:: - - Add callbacks + callbacks Module Reference diff --git a/neutron_lib/_callbacks/__init__.py b/neutron_lib/_callbacks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/neutron_lib/_callbacks/events.py b/neutron_lib/_callbacks/events.py new file mode 100644 index 000000000..7dfd83d5e --- /dev/null +++ b/neutron_lib/_callbacks/events.py @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# String literals representing core events. +BEFORE_CREATE = 'before_create' +BEFORE_READ = 'before_read' +BEFORE_UPDATE = 'before_update' +BEFORE_DELETE = 'before_delete' + +AFTER_CREATE = 'after_create' +AFTER_READ = 'after_read' +AFTER_UPDATE = 'after_update' +AFTER_DELETE = 'after_delete' + +ABORT_CREATE = 'abort_create' +ABORT_READ = 'abort_read' +ABORT_UPDATE = 'abort_update' +ABORT_DELETE = 'abort_delete' + +ABORT = 'abort_' +BEFORE = 'before_' diff --git a/neutron_lib/_callbacks/exceptions.py b/neutron_lib/_callbacks/exceptions.py new file mode 100644 index 000000000..a90f5d4c8 --- /dev/null +++ b/neutron_lib/_callbacks/exceptions.py @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib._i18n import _ +from neutron_lib import exceptions + + +class Invalid(exceptions.NeutronException): + message = _("The value '%(value)s' for %(element)s is not valid.") + + +class CallbackFailure(Exception): + + def __init__(self, errors): + self.errors = errors + + def __str__(self): + if isinstance(self.errors, list): + return ','.join(str(error) for error in self.errors) + else: + return str(self.errors) + + +class NotificationError(object): + + def __init__(self, callback_id, error): + self.callback_id = callback_id + self.error = error + + def __str__(self): + return 'Callback %s failed with "%s"' % (self.callback_id, self.error) diff --git a/neutron_lib/_callbacks/manager.py b/neutron_lib/_callbacks/manager.py new file mode 100644 index 000000000..79fb827f7 --- /dev/null +++ b/neutron_lib/_callbacks/manager.py @@ -0,0 +1,162 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections + +from oslo_log import log as logging +from oslo_utils import reflection + +from neutron_lib._callbacks import events +from neutron_lib._callbacks import exceptions +from neutron_lib._i18n import _LE + +LOG = logging.getLogger(__name__) + + +class CallbacksManager(object): + """A callback system that allows objects to cooperate in a loose manner.""" + + def __init__(self): + self.clear() + + def subscribe(self, callback, resource, event): + """Subscribe callback for a resource event. + + The same callback may register for more than one event. + + :param callback: the callback. It must raise or return a boolean. + :param resource: the resource. It must be a valid resource. + :param event: the event. It must be a valid event. + """ + LOG.debug("Subscribe: %(callback)s %(resource)s %(event)s", + {'callback': callback, 'resource': resource, 'event': event}) + + callback_id = _get_id(callback) + try: + self._callbacks[resource][event][callback_id] = callback + except KeyError: + # Initialize the registry for unknown resources and/or events + # prior to enlisting the callback. + self._callbacks[resource][event] = {} + self._callbacks[resource][event][callback_id] = callback + # We keep a copy of callbacks to speed the unsubscribe operation. + if callback_id not in self._index: + self._index[callback_id] = collections.defaultdict(set) + self._index[callback_id][resource].add(event) + + def unsubscribe(self, callback, resource, event): + """Unsubscribe callback from the registry. + + :param callback: the callback. + :param resource: the resource. + :param event: the event. + """ + LOG.debug("Unsubscribe: %(callback)s %(resource)s %(event)s", + {'callback': callback, 'resource': resource, 'event': event}) + + callback_id = self._find(callback) + if not callback_id: + LOG.debug("Callback %s not found", callback_id) + return + if resource and event: + del self._callbacks[resource][event][callback_id] + self._index[callback_id][resource].discard(event) + if not self._index[callback_id][resource]: + del self._index[callback_id][resource] + if not self._index[callback_id]: + del self._index[callback_id] + else: + value = '%s,%s' % (resource, event) + raise exceptions.Invalid(element='resource,event', value=value) + + def unsubscribe_by_resource(self, callback, resource): + """Unsubscribe callback for any event associated to the resource. + + :param callback: the callback. + :param resource: the resource. + """ + callback_id = self._find(callback) + if callback_id: + if resource in self._index[callback_id]: + for event in self._index[callback_id][resource]: + del self._callbacks[resource][event][callback_id] + del self._index[callback_id][resource] + if not self._index[callback_id]: + del self._index[callback_id] + + def unsubscribe_all(self, callback): + """Unsubscribe callback for all events and all resources. + + + :param callback: the callback. + """ + callback_id = self._find(callback) + if callback_id: + for resource, resource_events in self._index[callback_id].items(): + for event in resource_events: + del self._callbacks[resource][event][callback_id] + del self._index[callback_id] + + def notify(self, resource, event, trigger, **kwargs): + """Notify all subscribed callback(s). + + Dispatch the resource's event to the subscribed callbacks. + + :param resource: the resource. + :param event: the event. + :param trigger: the trigger. A reference to the sender of the event. + """ + errors = self._notify_loop(resource, event, trigger, **kwargs) + if errors and event.startswith(events.BEFORE): + abort_event = event.replace( + events.BEFORE, events.ABORT) + self._notify_loop(resource, abort_event, trigger) + raise exceptions.CallbackFailure(errors=errors) + + def clear(self): + """Brings the manager to a clean slate.""" + self._callbacks = collections.defaultdict(dict) + self._index = collections.defaultdict(dict) + + def _notify_loop(self, resource, event, trigger, **kwargs): + """The notification loop.""" + LOG.debug("Notify callbacks for %(resource)s, %(event)s", + {'resource': resource, 'event': event}) + + errors = [] + callbacks = self._callbacks[resource].get(event, {}).items() + # TODO(armax): consider using a GreenPile + for callback_id, callback in callbacks: + try: + LOG.debug("Calling callback %s", callback_id) + callback(resource, event, trigger, **kwargs) + except Exception as e: + LOG.exception(_LE("Error during notification for " + "%(callback)s %(resource)s, %(event)s"), + {'callback': callback_id, + 'resource': resource, + 'event': event}) + errors.append(exceptions.NotificationError(callback_id, e)) + return errors + + def _find(self, callback): + """Return the callback_id if found, None otherwise.""" + callback_id = _get_id(callback) + return callback_id if callback_id in self._index else None + + +def _get_id(callback): + """Return a unique identifier for the callback.""" + # TODO(armax): consider using something other than names + # https://www.python.org/dev/peps/pep-3155/, but this + # might be okay for now. + return reflection.get_callable_name(callback) diff --git a/neutron_lib/_callbacks/registry.py b/neutron_lib/_callbacks/registry.py new file mode 100644 index 000000000..6644f44df --- /dev/null +++ b/neutron_lib/_callbacks/registry.py @@ -0,0 +1,48 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron_lib._callbacks import manager + + +# TODO(armax): consider adding locking +CALLBACK_MANAGER = None + + +def _get_callback_manager(): + global CALLBACK_MANAGER + if CALLBACK_MANAGER is None: + CALLBACK_MANAGER = manager.CallbacksManager() + return CALLBACK_MANAGER + + +def subscribe(callback, resource, event): + _get_callback_manager().subscribe(callback, resource, event) + + +def unsubscribe(callback, resource, event): + _get_callback_manager().unsubscribe(callback, resource, event) + + +def unsubscribe_by_resource(callback, resource): + _get_callback_manager().unsubscribe_by_resource(callback, resource) + + +def unsubscribe_all(callback): + _get_callback_manager().unsubscribe_all(callback) + + +def notify(resource, event, trigger, **kwargs): + _get_callback_manager().notify(resource, event, trigger, **kwargs) + + +def clear(): + _get_callback_manager().clear() diff --git a/neutron_lib/_callbacks/resources.py b/neutron_lib/_callbacks/resources.py new file mode 100644 index 000000000..a0fd4c09e --- /dev/null +++ b/neutron_lib/_callbacks/resources.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# String literals representing core resources. +PORT = 'port' +PROCESS = 'process' +ROUTER = 'router' +ROUTER_GATEWAY = 'router_gateway' +ROUTER_INTERFACE = 'router_interface' +SECURITY_GROUP = 'security_group' +SECURITY_GROUP_RULE = 'security_group_rule' +SUBNET = 'subnet' +SUBNET_GATEWAY = 'subnet_gateway' diff --git a/neutron_lib/tests/unit/__init__.py b/neutron_lib/tests/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/neutron_lib/tests/unit/callbacks/__init__.py b/neutron_lib/tests/unit/callbacks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/neutron_lib/tests/unit/callbacks/test_callback_exceptions.py b/neutron_lib/tests/unit/callbacks/test_callback_exceptions.py new file mode 100644 index 000000000..dd60ba0fd --- /dev/null +++ b/neutron_lib/tests/unit/callbacks/test_callback_exceptions.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +test_callback_exceptions +---------------------------------- + +Tests for `neutron_lib.callback.exceptions` module. +""" + +import functools + +import neutron_lib._callbacks.exceptions as ex +from neutron_lib.tests import test_exceptions as base + + +class TestCallbackExceptions(base.TestExceptions): + + def _check_exception(self, exc_class, expected_msg, **kwargs): + raise_exc_class = functools.partial(base._raise, exc_class) + e = self.assertRaises(exc_class, raise_exc_class, **kwargs) + self.assertEqual(expected_msg, str(e)) + + def test_invalid(self): + self._check_exception( + ex.Invalid, + "The value 'foo' for bar is not valid.", + value='foo', element='bar') + + def test_callback_failure(self): + self._check_exception( + ex.CallbackFailure, + 'one', + errors='one') + + def test_callback_failure_with_list(self): + self._check_exception( + ex.CallbackFailure, + '1,2,3', + errors=[1, 2, 3]) + + def test_notification_error(self): + '''Test that correct message is create for this error class.''' + error = ex.NotificationError('abc', 'boom') + self.assertEqual('Callback abc failed with "boom"', str(error)) diff --git a/neutron_lib/tests/unit/callbacks/test_manager.py b/neutron_lib/tests/unit/callbacks/test_manager.py new file mode 100644 index 000000000..87f89361e --- /dev/null +++ b/neutron_lib/tests/unit/callbacks/test_manager.py @@ -0,0 +1,218 @@ +# Copyright 2015 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from oslotest import base + +from neutron_lib._callbacks import events +from neutron_lib._callbacks import exceptions +from neutron_lib._callbacks import manager +from neutron_lib._callbacks import resources + + +def callback_1(*args, **kwargs): + callback_1.counter += 1 +callback_id_1 = manager._get_id(callback_1) + + +def callback_2(*args, **kwargs): + callback_2.counter += 1 +callback_id_2 = manager._get_id(callback_2) + + +def callback_raise(*args, **kwargs): + raise Exception() + + +class CallBacksManagerTestCase(base.BaseTestCase): + + def setUp(self): + super(CallBacksManagerTestCase, self).setUp() + self.manager = manager.CallbacksManager() + callback_1.counter = 0 + callback_2.counter = 0 + + def test_subscribe(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.assertIsNotNone( + self.manager._callbacks[resources.PORT][events.BEFORE_CREATE]) + self.assertIn(callback_id_1, self.manager._index) + self.assertEqual(self.__module__ + '.callback_1', callback_id_1) + + def test_subscribe_unknown(self): + self.manager.subscribe( + callback_1, 'my_resource', 'my-event') + self.assertIsNotNone( + self.manager._callbacks['my_resource']['my-event']) + self.assertIn(callback_id_1, self.manager._index) + + def test_subscribe_is_idempotent(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.assertEqual( + 1, + len(self.manager._callbacks[resources.PORT][events.BEFORE_CREATE])) + callbacks = self.manager._index[callback_id_1][resources.PORT] + self.assertEqual(1, len(callbacks)) + + def test_subscribe_multiple_callbacks(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.subscribe( + callback_2, resources.PORT, events.BEFORE_CREATE) + self.assertEqual(2, len(self.manager._index)) + self.assertEqual( + 2, + len(self.manager._callbacks[resources.PORT][events.BEFORE_CREATE])) + + def test_unsubscribe(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.unsubscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.assertNotIn( + callback_id_1, + self.manager._callbacks[resources.PORT][events.BEFORE_CREATE]) + self.assertNotIn(callback_id_1, self.manager._index) + + def test_unsubscribe_unknown_callback(self): + self.manager.subscribe( + callback_2, resources.PORT, events.BEFORE_CREATE) + self.manager.unsubscribe(callback_1, mock.ANY, mock.ANY) + self.assertEqual(1, len(self.manager._index)) + + def test_fail_to_unsubscribe(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.assertRaises(exceptions.Invalid, + self.manager.unsubscribe, + callback_1, resources.PORT, None) + self.assertRaises(exceptions.Invalid, + self.manager.unsubscribe, + callback_1, None, events.BEFORE_CREATE) + + def test_unsubscribe_is_idempotent(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.unsubscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.unsubscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.assertNotIn(callback_id_1, self.manager._index) + self.assertNotIn(callback_id_1, + self.manager._callbacks[resources.PORT][events.BEFORE_CREATE]) + + def test_unsubscribe_by_resource(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_DELETE) + self.manager.subscribe( + callback_2, resources.PORT, events.BEFORE_DELETE) + self.manager.unsubscribe_by_resource(callback_1, resources.PORT) + self.assertNotIn( + callback_id_1, + self.manager._callbacks[resources.PORT][events.BEFORE_CREATE]) + self.assertIn( + callback_id_2, + self.manager._callbacks[resources.PORT][events.BEFORE_DELETE]) + self.assertNotIn(callback_id_1, self.manager._index) + + def test_unsubscribe_all(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_DELETE) + self.manager.subscribe( + callback_1, resources.ROUTER, events.BEFORE_CREATE) + self.manager.unsubscribe_all(callback_1) + self.assertNotIn( + callback_id_1, + self.manager._callbacks[resources.PORT][events.BEFORE_CREATE]) + self.assertNotIn(callback_id_1, self.manager._index) + + def test_notify_none(self): + self.manager.notify(resources.PORT, events.BEFORE_CREATE, mock.ANY) + self.assertEqual(0, callback_1.counter) + self.assertEqual(0, callback_2.counter) + + def test_feebly_referenced_callback(self): + self.manager.subscribe(lambda *x, **y: None, resources.PORT, + events.BEFORE_CREATE) + self.manager.notify(resources.PORT, events.BEFORE_CREATE, mock.ANY) + + def test_notify_with_exception(self): + with mock.patch.object(self.manager, '_notify_loop') as n: + n.return_value = ['error'] + self.assertRaises(exceptions.CallbackFailure, + self.manager.notify, + mock.ANY, events.BEFORE_CREATE, mock.ANY) + expected_calls = [ + mock.call(mock.ANY, 'before_create', mock.ANY), + mock.call(mock.ANY, 'abort_create', mock.ANY) + ] + n.assert_has_calls(expected_calls) + + def test_notify_handle_exception(self): + self.manager.subscribe( + callback_raise, resources.PORT, events.BEFORE_CREATE) + e = self.assertRaises(exceptions.CallbackFailure, self.manager.notify, + resources.PORT, events.BEFORE_CREATE, self) + self.assertIsInstance(e.errors[0], exceptions.NotificationError) + + def test_notify_called_once_with_no_failures(self): + with mock.patch.object(self.manager, '_notify_loop') as n: + n.return_value = False + self.manager.notify(resources.PORT, events.BEFORE_CREATE, mock.ANY) + n.assert_called_once_with( + resources.PORT, events.BEFORE_CREATE, mock.ANY) + + def test__notify_loop_single_event(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.subscribe( + callback_2, resources.PORT, events.BEFORE_CREATE) + self.manager._notify_loop( + resources.PORT, events.BEFORE_CREATE, mock.ANY) + self.assertEqual(1, callback_1.counter) + self.assertEqual(1, callback_2.counter) + + def test__notify_loop_multiple_events(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.subscribe( + callback_1, resources.ROUTER, events.BEFORE_DELETE) + self.manager.subscribe( + callback_2, resources.PORT, events.BEFORE_CREATE) + self.manager._notify_loop( + resources.PORT, events.BEFORE_CREATE, mock.ANY) + self.manager._notify_loop( + resources.ROUTER, events.BEFORE_DELETE, mock.ANY) + self.assertEqual(2, callback_1.counter) + self.assertEqual(1, callback_2.counter) + + def test_clearing_subscribers(self): + self.manager.subscribe( + callback_1, resources.PORT, events.BEFORE_CREATE) + self.manager.subscribe( + callback_2, resources.PORT, events.AFTER_CREATE) + self.assertEqual(2, len(self.manager._callbacks[resources.PORT])) + self.assertEqual(2, len(self.manager._index)) + self.manager.clear() + self.assertEqual(0, len(self.manager._callbacks)) + self.assertEqual(0, len(self.manager._index)) diff --git a/neutron_lib/tests/unit/callbacks/test_registry.py b/neutron_lib/tests/unit/callbacks/test_registry.py new file mode 100644 index 000000000..3b03e048a --- /dev/null +++ b/neutron_lib/tests/unit/callbacks/test_registry.py @@ -0,0 +1,59 @@ +# Copyright 2015 Cisco Systems Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from oslotest import base + +from neutron_lib._callbacks import registry + + +def my_callback(): + pass + + +class TestCallbackRegistryDispatching(base.BaseTestCase): + + def setUp(self): + super(TestCallbackRegistryDispatching, self).setUp() + registry.CALLBACK_MANAGER = mock.Mock() + + def test_subscribe(self): + registry.subscribe(my_callback, 'my-resource', 'my-event') + registry.CALLBACK_MANAGER.subscribe.assert_called_with( + my_callback, 'my-resource', 'my-event') + + def test_unsubscribe(self): + registry.unsubscribe(my_callback, 'my-resource', 'my-event') + registry.CALLBACK_MANAGER.unsubscribe.assert_called_with( + my_callback, 'my-resource', 'my-event') + + def test_unsubscribe_by_resource(self): + registry.unsubscribe_by_resource(my_callback, 'my-resource') + registry.CALLBACK_MANAGER.unsubscribe_by_resource.assert_called_with( + my_callback, 'my-resource') + + def test_unsubscribe_all(self): + registry.unsubscribe_all(my_callback) + registry.CALLBACK_MANAGER.unsubscribe_all.assert_called_with( + my_callback) + + def test_notify(self): + registry.notify('my-resource', 'my-event', mock.ANY) + registry.CALLBACK_MANAGER.notify.assert_called_with( + 'my-resource', 'my-event', mock.ANY) + + def test_clear(self): + registry.clear() + registry.CALLBACK_MANAGER.clear.assert_called_with() diff --git a/requirements.txt b/requirements.txt index 363a08fc5..540af2c95 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ pbr>=1.6 Babel>=1.3 oslo.i18n>=1.5.0 # Apache-2.0 +oslo.log>=1.12.0 # Apache-2.0 oslo.utils>=2.8.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 1d3ff2bef..c19113c0b 100644 --- a/tox.ini +++ b/tox.ini @@ -28,8 +28,10 @@ commands = oslo_debug_helper {posargs} [flake8] # E123, E125 skipped as they are invalid PEP-8. +# E126 continuation line over-indented for hanging indent +# E128 continuation line under-indented for visual indent show-source = True -ignore = E123,E125 +ignore = E123,E125,E126,E128 builtins = _ exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build