00d07f5205
This change formalises locking in MessageHandlingServer, which closes several bugs: * It adds locking for internal state when using the blocking executor, which closes a number of races. * It does not hold a lock while executing server functions, which removes a potential cause of deadlock if the server does its own locking. * It fixes a regression introduced in change gI3cfbe1bf02d451e379b1dcc23dacb0139c03be76. If multiple threads called wait() simultaneously, only 1 of them would wait and the others would return immediately, despite message handling not having completed. With this change only 1 will call the underlying wait, but all will wait on its completion. Additionally, it introduces some new functionality: * It allows the user to make calls in any order and it will ensure, with locking, that these will be reordered appropriately. * The caller can pass a `timeout` argument to any server method, which will cause it to raise an exception if it waits too long. * The caller can pass a `log_after` argument to any server method, which will cause it to raise a log message if it waits too long. It can also be used to disable logging when waiting is intentional. We remove DummyCondition as it no longer has any users. This change was originally committed as change I9d516b208446963dcd80b75e2d5a2cecb1187efa, but was reverted as it caused a hang in a Nova test. This was caused by the locking behaviour for handling restarting a previously stopped server. The original patch caused the state to 'wrap' immediately after the user called wait(). This caused a hang in tests which redundantly called stop() and wait() multiple times. This new patch only wraps when the user calls start() again. Callers who do not restart a server will therefore not be affected by the wrapping behaviour. Callers who do restart a server will be no worse than before. We add a deprecation warning on restart, as this operation is inherently racy with this api and there is a simple, safe alternative. This new version has been successfully tested against the unit and functional tests of nova, cinder, glance, and ceilometer. Change-Id: Ic79f87e7b069c1f62d6121486fd6cafd732fdde7
131 lines
4.1 KiB
Python
131 lines
4.1 KiB
Python
|
|
# Copyright 2013 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import logging
|
|
import threading
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
def version_is_compatible(imp_version, version):
|
|
"""Determine whether versions are compatible.
|
|
|
|
:param imp_version: The version implemented
|
|
:param version: The version requested by an incoming message.
|
|
"""
|
|
version_parts = version.split('.')
|
|
imp_version_parts = imp_version.split('.')
|
|
try:
|
|
rev = version_parts[2]
|
|
except IndexError:
|
|
rev = 0
|
|
try:
|
|
imp_rev = imp_version_parts[2]
|
|
except IndexError:
|
|
imp_rev = 0
|
|
|
|
if int(version_parts[0]) != int(imp_version_parts[0]): # Major
|
|
return False
|
|
if int(version_parts[1]) > int(imp_version_parts[1]): # Minor
|
|
return False
|
|
if (int(version_parts[1]) == int(imp_version_parts[1]) and
|
|
int(rev) > int(imp_rev)): # Revision
|
|
return False
|
|
return True
|
|
|
|
|
|
class DispatcherExecutorContext(object):
|
|
"""Dispatcher executor context helper
|
|
|
|
A dispatcher can have work to do before and after the dispatch of the
|
|
request in the main server thread while the dispatcher itself can be
|
|
done in its own thread.
|
|
|
|
The executor can use the helper like this:
|
|
|
|
callback = dispatcher(incoming)
|
|
callback.prepare()
|
|
thread = MyWhateverThread()
|
|
thread.on_done(callback.done)
|
|
thread.run(callback.run)
|
|
|
|
"""
|
|
def __init__(self, incoming, dispatch, executor_callback=None,
|
|
post=None):
|
|
self._result = None
|
|
self._incoming = incoming
|
|
self._dispatch = dispatch
|
|
self._post = post
|
|
self._executor_callback = executor_callback
|
|
|
|
def run(self):
|
|
"""The incoming message dispath itself
|
|
|
|
Can be run in an other thread/greenlet/corotine if the executor is
|
|
able to do it.
|
|
"""
|
|
try:
|
|
self._result = self._dispatch(self._incoming,
|
|
self._executor_callback)
|
|
except Exception:
|
|
msg = 'The dispatcher method must catches all exceptions'
|
|
LOG.exception(msg)
|
|
raise RuntimeError(msg)
|
|
|
|
def done(self):
|
|
"""Callback after the incoming message have been dispathed
|
|
|
|
Should be ran in the main executor thread/greenlet/corotine
|
|
"""
|
|
# FIXME(sileht): this is not currently true, this works only because
|
|
# the driver connection used for polling write on the wire only to
|
|
# ack/requeue message, but what if one day, the driver do something
|
|
# else
|
|
if self._post is not None:
|
|
self._post(self._incoming, self._result)
|
|
|
|
|
|
def fetch_current_thread_functor():
|
|
# Until https://github.com/eventlet/eventlet/issues/172 is resolved
|
|
# or addressed we have to use complicated workaround to get a object
|
|
# that will not be recycled; the usage of threading.current_thread()
|
|
# doesn't appear to currently be monkey patched and therefore isn't
|
|
# reliable to use (and breaks badly when used as all threads share
|
|
# the same current_thread() object)...
|
|
try:
|
|
import eventlet
|
|
from eventlet import patcher
|
|
green_threaded = patcher.is_monkey_patched('thread')
|
|
except ImportError:
|
|
green_threaded = False
|
|
if green_threaded:
|
|
return lambda: eventlet.getcurrent()
|
|
else:
|
|
return lambda: threading.current_thread()
|
|
|
|
|
|
class DummyLock(object):
|
|
def acquire(self):
|
|
pass
|
|
|
|
def release(self):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
self.acquire()
|
|
|
|
def __exit__(self, type, value, traceback):
|
|
self.release()
|