985 lines
36 KiB
Python
985 lines
36 KiB
Python
# Copyright 2013 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import datetime
|
|
import ssl
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
import fixtures
|
|
import kombu
|
|
import kombu.transport.memory
|
|
from oslo_config import cfg
|
|
from oslo_serialization import jsonutils
|
|
from oslotest import mockpatch
|
|
import testscenarios
|
|
|
|
import oslo_messaging
|
|
from oslo_messaging._drivers import amqp
|
|
from oslo_messaging._drivers import amqpdriver
|
|
from oslo_messaging._drivers import common as driver_common
|
|
from oslo_messaging._drivers import impl_rabbit as rabbit_driver
|
|
from oslo_messaging.tests import utils as test_utils
|
|
from six.moves import mock
|
|
|
|
load_tests = testscenarios.load_tests_apply_scenarios
|
|
|
|
|
|
class TestDeprecatedRabbitDriverLoad(test_utils.BaseTestCase):
|
|
|
|
def setUp(self):
|
|
super(TestDeprecatedRabbitDriverLoad, self).setUp(
|
|
conf=cfg.ConfigOpts())
|
|
self.messaging_conf.transport_driver = 'rabbit'
|
|
self.config(fake_rabbit=True, group="oslo_messaging_rabbit")
|
|
|
|
def test_driver_load(self):
|
|
transport = oslo_messaging.get_transport(self.conf)
|
|
self.addCleanup(transport.cleanup)
|
|
driver = transport._driver
|
|
url = driver._get_connection()._url
|
|
|
|
self.assertIsInstance(driver, rabbit_driver.RabbitDriver)
|
|
self.assertEqual('memory:////', url)
|
|
|
|
|
|
class TestHeartbeat(test_utils.BaseTestCase):
|
|
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.LOG')
|
|
@mock.patch('kombu.connection.Connection.heartbeat_check')
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.'
|
|
'_heartbeat_supported_and_enabled', return_value=True)
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.'
|
|
'ensure_connection')
|
|
def _do_test_heartbeat_sent(self, fake_ensure_connection,
|
|
fake_heartbeat_support, fake_heartbeat,
|
|
fake_logger, heartbeat_side_effect=None,
|
|
info=None):
|
|
|
|
event = threading.Event()
|
|
|
|
def heartbeat_check(rate=2):
|
|
event.set()
|
|
if heartbeat_side_effect:
|
|
raise heartbeat_side_effect
|
|
|
|
fake_heartbeat.side_effect = heartbeat_check
|
|
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
conn = transport._driver._get_connection()
|
|
conn.ensure(method=lambda: True)
|
|
event.wait()
|
|
conn._heartbeat_stop()
|
|
|
|
# check heartbeat have been called
|
|
self.assertLess(0, fake_heartbeat.call_count)
|
|
|
|
if not heartbeat_side_effect:
|
|
self.assertEqual(1, fake_ensure_connection.call_count)
|
|
self.assertEqual(2, fake_logger.info.call_count)
|
|
else:
|
|
self.assertEqual(2, fake_ensure_connection.call_count)
|
|
self.assertEqual(3, fake_logger.info.call_count)
|
|
self.assertIn(mock.call(info, mock.ANY),
|
|
fake_logger.info.mock_calls)
|
|
|
|
def test_test_heartbeat_sent_default(self):
|
|
self._do_test_heartbeat_sent()
|
|
|
|
def test_test_heartbeat_sent_connection_fail(self):
|
|
self._do_test_heartbeat_sent(
|
|
heartbeat_side_effect=kombu.exceptions.ConnectionError,
|
|
info='A recoverable connection/channel error occurred, '
|
|
'trying to reconnect: %s')
|
|
|
|
|
|
class TestRabbitDriverLoad(test_utils.BaseTestCase):
|
|
|
|
scenarios = [
|
|
('rabbit', dict(transport_driver='rabbit',
|
|
url='amqp://guest:guest@localhost:5672//')),
|
|
('kombu', dict(transport_driver='kombu',
|
|
url='amqp://guest:guest@localhost:5672//')),
|
|
('rabbit+memory', dict(transport_driver='kombu+memory',
|
|
url='memory:///'))
|
|
]
|
|
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.ensure')
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.reset')
|
|
def test_driver_load(self, fake_ensure, fake_reset):
|
|
self.config(heartbeat_timeout_threshold=60,
|
|
group='oslo_messaging_rabbit')
|
|
self.messaging_conf.transport_driver = self.transport_driver
|
|
transport = oslo_messaging.get_transport(self.conf)
|
|
self.addCleanup(transport.cleanup)
|
|
driver = transport._driver
|
|
url = driver._get_connection()._url
|
|
|
|
self.assertIsInstance(driver, rabbit_driver.RabbitDriver)
|
|
self.assertEqual(self.url, url)
|
|
|
|
|
|
class TestRabbitDriverLoadSSL(test_utils.BaseTestCase):
|
|
scenarios = [
|
|
('no_ssl', dict(options=dict(), expected=False)),
|
|
('no_ssl_with_options', dict(options=dict(kombu_ssl_version='TLSv1'),
|
|
expected=False)),
|
|
('just_ssl', dict(options=dict(rabbit_use_ssl=True),
|
|
expected=True)),
|
|
('ssl_with_options', dict(options=dict(rabbit_use_ssl=True,
|
|
kombu_ssl_version='TLSv1',
|
|
kombu_ssl_keyfile='foo',
|
|
kombu_ssl_certfile='bar',
|
|
kombu_ssl_ca_certs='foobar'),
|
|
expected=dict(ssl_version=3,
|
|
keyfile='foo',
|
|
certfile='bar',
|
|
ca_certs='foobar',
|
|
cert_reqs=ssl.CERT_REQUIRED))),
|
|
]
|
|
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.ensure')
|
|
@mock.patch('kombu.connection.Connection')
|
|
def test_driver_load(self, connection_klass, fake_ensure):
|
|
self.config(group="oslo_messaging_rabbit", **self.options)
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
|
|
transport._driver._get_connection()
|
|
connection_klass.assert_called_once_with(
|
|
'memory:///', transport_options={'confirm_publish': True,
|
|
'on_blocked': mock.ANY,
|
|
'on_unblocked': mock.ANY},
|
|
ssl=self.expected, login_method='AMQPLAIN',
|
|
heartbeat=60, failover_strategy="shuffle")
|
|
|
|
|
|
class TestRabbitPublisher(test_utils.BaseTestCase):
|
|
@mock.patch('kombu.messaging.Producer.publish')
|
|
def test_send_with_timeout(self, fake_publish):
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
with transport._driver._get_connection(amqp.PURPOSE_SEND) as pool_conn:
|
|
conn = pool_conn.connection
|
|
conn._publish(mock.Mock(), 'msg', routing_key='routing_key',
|
|
timeout=1)
|
|
fake_publish.assert_called_with('msg', expiration=1000)
|
|
|
|
@mock.patch('kombu.messaging.Producer.publish')
|
|
def test_send_no_timeout(self, fake_publish):
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
with transport._driver._get_connection(amqp.PURPOSE_SEND) as pool_conn:
|
|
conn = pool_conn.connection
|
|
conn._publish(mock.Mock(), 'msg', routing_key='routing_key')
|
|
fake_publish.assert_called_with('msg', expiration=None)
|
|
|
|
def test_declared_queue_publisher(self):
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
|
|
e_passive = kombu.entity.Exchange(
|
|
name='foobar',
|
|
type='topic',
|
|
passive=True)
|
|
|
|
e_active = kombu.entity.Exchange(
|
|
name='foobar',
|
|
type='topic',
|
|
passive=False)
|
|
|
|
with transport._driver._get_connection(amqp.PURPOSE_SEND) as pool_conn:
|
|
conn = pool_conn.connection
|
|
exc = conn.connection.channel_errors[0]
|
|
|
|
def try_send(exchange):
|
|
conn._ensure_publishing(
|
|
conn._publish_and_creates_default_queue,
|
|
exchange, {}, routing_key='foobar')
|
|
|
|
# Ensure the exchange does not exists
|
|
self.assertRaises(exc, try_send, e_passive)
|
|
# Create it
|
|
try_send(e_active)
|
|
# Ensure it creates it
|
|
try_send(e_passive)
|
|
|
|
with mock.patch('kombu.messaging.Producer', side_effect=exc):
|
|
# Should reset the cache and ensures the exchange does
|
|
# not exists
|
|
self.assertRaises(exc, try_send, e_passive)
|
|
# Recreate it
|
|
try_send(e_active)
|
|
# Ensure it have been recreated
|
|
try_send(e_passive)
|
|
|
|
|
|
class TestRabbitConsume(test_utils.BaseTestCase):
|
|
|
|
def test_consume_timeout(self):
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
deadline = time.time() + 6
|
|
with transport._driver._get_connection(amqp.PURPOSE_LISTEN) as conn:
|
|
self.assertRaises(driver_common.Timeout,
|
|
conn.consume, timeout=3)
|
|
|
|
# kombu memory transport doesn't really raise error
|
|
# so just simulate a real driver behavior
|
|
conn.connection.connection.recoverable_channel_errors = (IOError,)
|
|
conn.declare_fanout_consumer("notif.info", lambda msg: True)
|
|
with mock.patch('kombu.connection.Connection.drain_events',
|
|
side_effect=IOError):
|
|
self.assertRaises(driver_common.Timeout,
|
|
conn.consume, timeout=3)
|
|
|
|
self.assertEqual(0, int(deadline - time.time()))
|
|
|
|
def test_connection_ack_have_disconnected_kombu_connection(self):
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
with transport._driver._get_connection(amqp.PURPOSE_LISTEN) as conn:
|
|
channel = conn.connection.channel
|
|
with mock.patch('kombu.connection.Connection.connected',
|
|
new_callable=mock.PropertyMock,
|
|
return_value=False):
|
|
self.assertRaises(driver_common.Timeout,
|
|
conn.connection.consume, timeout=0.01)
|
|
# Ensure a new channel have been setuped
|
|
self.assertNotEqual(channel, conn.connection.channel)
|
|
|
|
|
|
class TestRabbitTransportURL(test_utils.BaseTestCase):
|
|
|
|
scenarios = [
|
|
('none', dict(url=None,
|
|
expected=["amqp://guest:guest@localhost:5672//"])),
|
|
('memory', dict(url='kombu+memory:////',
|
|
expected=["memory:///"])),
|
|
('empty',
|
|
dict(url='rabbit:///',
|
|
expected=['amqp://guest:guest@localhost:5672/'])),
|
|
('localhost',
|
|
dict(url='rabbit://localhost/',
|
|
expected=['amqp://:@localhost:5672/'])),
|
|
('virtual_host',
|
|
dict(url='rabbit:///vhost',
|
|
expected=['amqp://guest:guest@localhost:5672/vhost'])),
|
|
('no_creds',
|
|
dict(url='rabbit://host/virtual_host',
|
|
expected=['amqp://:@host:5672/virtual_host'])),
|
|
('no_port',
|
|
dict(url='rabbit://user:password@host/virtual_host',
|
|
expected=['amqp://user:password@host:5672/virtual_host'])),
|
|
('full_url',
|
|
dict(url='rabbit://user:password@host:10/virtual_host',
|
|
expected=['amqp://user:password@host:10/virtual_host'])),
|
|
('full_two_url',
|
|
dict(url='rabbit://user:password@host:10,'
|
|
'user2:password2@host2:12/virtual_host',
|
|
expected=["amqp://user:password@host:10/virtual_host",
|
|
"amqp://user2:password2@host2:12/virtual_host"]
|
|
)),
|
|
('rabbit_ipv6',
|
|
dict(url='rabbit://u:p@[fd00:beef:dead:55::133]:10/vhost',
|
|
expected=['amqp://u:p@[fd00:beef:dead:55::133]:10/vhost'])),
|
|
('rabbit_ipv4',
|
|
dict(url='rabbit://user:password@10.20.30.40:10/vhost',
|
|
expected=['amqp://user:password@10.20.30.40:10/vhost'])),
|
|
]
|
|
|
|
def setUp(self):
|
|
super(TestRabbitTransportURL, self).setUp()
|
|
self.messaging_conf.transport_driver = 'rabbit'
|
|
self.config(heartbeat_timeout_threshold=0,
|
|
group='oslo_messaging_rabbit')
|
|
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.ensure')
|
|
@mock.patch('oslo_messaging._drivers.impl_rabbit.Connection.reset')
|
|
def test_transport_url(self, fake_reset, fake_ensure):
|
|
transport = oslo_messaging.get_transport(self.conf, self.url)
|
|
self.addCleanup(transport.cleanup)
|
|
driver = transport._driver
|
|
|
|
urls = driver._get_connection()._url.split(";")
|
|
self.assertEqual(sorted(self.expected), sorted(urls))
|
|
|
|
|
|
class TestSendReceive(test_utils.BaseTestCase):
|
|
|
|
_n_senders = [
|
|
('single_sender', dict(n_senders=1)),
|
|
('multiple_senders', dict(n_senders=10)),
|
|
]
|
|
|
|
_context = [
|
|
('empty_context', dict(ctxt={})),
|
|
('with_context', dict(ctxt={'user': 'mark'})),
|
|
]
|
|
|
|
_reply = [
|
|
('rx_id', dict(rx_id=True, reply=None)),
|
|
('none', dict(rx_id=False, reply=None)),
|
|
('empty_list', dict(rx_id=False, reply=[])),
|
|
('empty_dict', dict(rx_id=False, reply={})),
|
|
('false', dict(rx_id=False, reply=False)),
|
|
('zero', dict(rx_id=False, reply=0)),
|
|
]
|
|
|
|
_reply_fail = [
|
|
('reply_success', dict(reply_failure_404=False)),
|
|
('reply_failure', dict(reply_failure_404=True)),
|
|
]
|
|
|
|
_failure = [
|
|
('success', dict(failure=False)),
|
|
('failure', dict(failure=True, expected=False)),
|
|
('expected_failure', dict(failure=True, expected=True)),
|
|
]
|
|
|
|
_timeout = [
|
|
('no_timeout', dict(timeout=None)),
|
|
('timeout', dict(timeout=0.01)), # FIXME(markmc): timeout=0 is broken?
|
|
]
|
|
|
|
_reply_ending = [
|
|
('old_behavior', dict(send_single_reply=False)),
|
|
('new_behavior', dict(send_single_reply=True)),
|
|
]
|
|
|
|
@classmethod
|
|
def generate_scenarios(cls):
|
|
cls.scenarios = testscenarios.multiply_scenarios(cls._n_senders,
|
|
cls._context,
|
|
cls._reply,
|
|
cls._reply_fail,
|
|
cls._failure,
|
|
cls._timeout,
|
|
cls._reply_ending)
|
|
|
|
def test_send_receive(self):
|
|
self.config(kombu_reconnect_timeout=0.5,
|
|
group="oslo_messaging_rabbit")
|
|
self.config(heartbeat_timeout_threshold=0,
|
|
group="oslo_messaging_rabbit")
|
|
self.config(send_single_reply=self.send_single_reply,
|
|
group="oslo_messaging_rabbit")
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
|
|
driver = transport._driver
|
|
|
|
target = oslo_messaging.Target(topic='testtopic')
|
|
|
|
listener = driver.listen(target)
|
|
|
|
senders = []
|
|
replies = []
|
|
msgs = []
|
|
errors = []
|
|
|
|
def stub_error(msg, *a, **kw):
|
|
if (a and len(a) == 1 and isinstance(a[0], dict) and a[0]):
|
|
a = a[0]
|
|
errors.append(str(msg) % a)
|
|
|
|
self.stubs.Set(driver_common.LOG, 'error', stub_error)
|
|
|
|
def send_and_wait_for_reply(i):
|
|
try:
|
|
if self.reply_failure_404:
|
|
timeout = 0.01
|
|
else:
|
|
timeout = self.timeout
|
|
replies.append(driver.send(target,
|
|
self.ctxt,
|
|
{'tx_id': i},
|
|
wait_for_reply=True,
|
|
timeout=timeout))
|
|
self.assertFalse(self.failure)
|
|
self.assertIsNone(self.timeout)
|
|
except (ZeroDivisionError, oslo_messaging.MessagingTimeout) as e:
|
|
replies.append(e)
|
|
self.assertTrue(self.failure or self.timeout is not None
|
|
or self.reply_failure_404)
|
|
|
|
while len(senders) < self.n_senders:
|
|
senders.append(threading.Thread(target=send_and_wait_for_reply,
|
|
args=(len(senders), )))
|
|
|
|
for i in range(len(senders)):
|
|
senders[i].start()
|
|
|
|
received = listener.poll()
|
|
self.assertIsNotNone(received)
|
|
self.assertEqual(self.ctxt, received.ctxt)
|
|
self.assertEqual({'tx_id': i}, received.message)
|
|
msgs.append(received)
|
|
|
|
# reply in reverse, except reply to the first guy second from last
|
|
order = list(range(len(senders) - 1, -1, -1))
|
|
if len(order) > 1:
|
|
order[-1], order[-2] = order[-2], order[-1]
|
|
|
|
if self.reply_failure_404:
|
|
start = time.time()
|
|
# NOTE(sileht): Simulate a rpc client restart
|
|
# By returning a ExchangeNotFound when we try to
|
|
# send reply
|
|
exc = (driver._reply_q_conn.connection.
|
|
connection.channel_errors[0]())
|
|
exc.code = 404
|
|
self.useFixture(mockpatch.Patch(
|
|
'kombu.messaging.Producer.publish',
|
|
side_effect=exc))
|
|
|
|
for i in order:
|
|
if self.timeout is None:
|
|
if self.failure:
|
|
try:
|
|
raise ZeroDivisionError
|
|
except Exception:
|
|
failure = sys.exc_info()
|
|
msgs[i].reply(failure=failure,
|
|
log_failure=not self.expected)
|
|
elif self.rx_id:
|
|
msgs[i].reply({'rx_id': i})
|
|
else:
|
|
msgs[i].reply(self.reply)
|
|
elif self.reply_failure_404:
|
|
msgs[i].reply({})
|
|
senders[i].join()
|
|
|
|
if self.reply_failure_404:
|
|
# NOTE(sileht) all reply fail, first take
|
|
# kombu_reconnect_timeout seconds to fail
|
|
# next immediately fail
|
|
dt = time.time() - start
|
|
timeout = self.conf.oslo_messaging_rabbit.kombu_reconnect_timeout
|
|
self.assertTrue(timeout <= dt < (timeout + 0.100), dt)
|
|
|
|
self.assertEqual(len(senders), len(replies))
|
|
for i, reply in enumerate(replies):
|
|
if self.timeout is not None or self.reply_failure_404:
|
|
self.assertIsInstance(reply, oslo_messaging.MessagingTimeout)
|
|
elif self.failure:
|
|
self.assertIsInstance(reply, ZeroDivisionError)
|
|
elif self.rx_id:
|
|
self.assertEqual({'rx_id': order[i]}, reply)
|
|
else:
|
|
self.assertEqual(self.reply, reply)
|
|
|
|
if not self.timeout and self.failure and not self.expected:
|
|
self.assertTrue(len(errors) > 0, errors)
|
|
else:
|
|
self.assertEqual(0, len(errors), errors)
|
|
|
|
|
|
TestSendReceive.generate_scenarios()
|
|
|
|
|
|
class TestPollAsync(test_utils.BaseTestCase):
|
|
|
|
def test_poll_timeout(self):
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
driver = transport._driver
|
|
target = oslo_messaging.Target(topic='testtopic')
|
|
listener = driver.listen(target)
|
|
received = listener.poll(timeout=0.050)
|
|
self.assertIsNone(received)
|
|
|
|
|
|
class TestRacyWaitForReply(test_utils.BaseTestCase):
|
|
|
|
def test_send_receive(self):
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
|
|
driver = transport._driver
|
|
|
|
target = oslo_messaging.Target(topic='testtopic')
|
|
|
|
listener = driver.listen(target)
|
|
|
|
senders = []
|
|
replies = []
|
|
msgs = []
|
|
|
|
wait_conditions = []
|
|
orig_reply_waiter = amqpdriver.ReplyWaiter.wait
|
|
|
|
def reply_waiter(self, msg_id, timeout):
|
|
if wait_conditions:
|
|
cond = wait_conditions.pop()
|
|
with cond:
|
|
cond.notify()
|
|
with cond:
|
|
cond.wait()
|
|
return orig_reply_waiter(self, msg_id, timeout)
|
|
|
|
self.stubs.Set(amqpdriver.ReplyWaiter, 'wait', reply_waiter)
|
|
|
|
def send_and_wait_for_reply(i, wait_for_reply):
|
|
replies.append(driver.send(target,
|
|
{},
|
|
{'tx_id': i},
|
|
wait_for_reply=wait_for_reply,
|
|
timeout=None))
|
|
|
|
while len(senders) < 2:
|
|
t = threading.Thread(target=send_and_wait_for_reply,
|
|
args=(len(senders), True))
|
|
t.daemon = True
|
|
senders.append(t)
|
|
|
|
# test the case then msg_id is not set
|
|
t = threading.Thread(target=send_and_wait_for_reply,
|
|
args=(len(senders), False))
|
|
t.daemon = True
|
|
senders.append(t)
|
|
|
|
# Start the first guy, receive his message, but delay his polling
|
|
notify_condition = threading.Condition()
|
|
wait_conditions.append(notify_condition)
|
|
with notify_condition:
|
|
senders[0].start()
|
|
notify_condition.wait()
|
|
|
|
msgs.append(listener.poll())
|
|
self.assertEqual({'tx_id': 0}, msgs[-1].message)
|
|
|
|
# Start the second guy, receive his message
|
|
senders[1].start()
|
|
|
|
msgs.append(listener.poll())
|
|
self.assertEqual({'tx_id': 1}, msgs[-1].message)
|
|
|
|
# Reply to both in order, making the second thread queue
|
|
# the reply meant for the first thread
|
|
msgs[0].reply({'rx_id': 0})
|
|
msgs[1].reply({'rx_id': 1})
|
|
|
|
# Wait for the second thread to finish
|
|
senders[1].join()
|
|
|
|
# Start the 3rd guy, receive his message
|
|
senders[2].start()
|
|
|
|
msgs.append(listener.poll())
|
|
self.assertEqual({'tx_id': 2}, msgs[-1].message)
|
|
|
|
# Verify the _send_reply was not invoked by driver:
|
|
with mock.patch.object(msgs[2], '_send_reply') as method:
|
|
msgs[2].reply({'rx_id': 2})
|
|
self.assertEqual(method.call_count, 0)
|
|
|
|
# Wait for the 3rd thread to finish
|
|
senders[2].join()
|
|
|
|
# Let the first thread continue
|
|
with notify_condition:
|
|
notify_condition.notify()
|
|
|
|
# Wait for the first thread to finish
|
|
senders[0].join()
|
|
|
|
# Verify replies were received out of order
|
|
self.assertEqual(len(senders), len(replies))
|
|
self.assertEqual({'rx_id': 1}, replies[0])
|
|
self.assertIsNone(replies[1])
|
|
self.assertEqual({'rx_id': 0}, replies[2])
|
|
|
|
|
|
def _declare_queue(target):
|
|
connection = kombu.connection.BrokerConnection(transport='memory')
|
|
|
|
# Kludge to speed up tests.
|
|
connection.transport.polling_interval = 0.0
|
|
|
|
connection.connect()
|
|
channel = connection.channel()
|
|
|
|
# work around 'memory' transport bug in 1.1.3
|
|
channel._new_queue('ae.undeliver')
|
|
|
|
if target.fanout:
|
|
exchange = kombu.entity.Exchange(name=target.topic + '_fanout',
|
|
type='fanout',
|
|
durable=False,
|
|
auto_delete=True)
|
|
queue = kombu.entity.Queue(name=target.topic + '_fanout_12345',
|
|
channel=channel,
|
|
exchange=exchange,
|
|
routing_key=target.topic)
|
|
elif target.server:
|
|
exchange = kombu.entity.Exchange(name='openstack',
|
|
type='topic',
|
|
durable=False,
|
|
auto_delete=False)
|
|
topic = '%s.%s' % (target.topic, target.server)
|
|
queue = kombu.entity.Queue(name=topic,
|
|
channel=channel,
|
|
exchange=exchange,
|
|
routing_key=topic)
|
|
else:
|
|
exchange = kombu.entity.Exchange(name='openstack',
|
|
type='topic',
|
|
durable=False,
|
|
auto_delete=False)
|
|
queue = kombu.entity.Queue(name=target.topic,
|
|
channel=channel,
|
|
exchange=exchange,
|
|
routing_key=target.topic)
|
|
|
|
queue.declare()
|
|
|
|
return connection, channel, queue
|
|
|
|
|
|
class TestRequestWireFormat(test_utils.BaseTestCase):
|
|
|
|
_target = [
|
|
('topic_target',
|
|
dict(topic='testtopic', server=None, fanout=False)),
|
|
('server_target',
|
|
dict(topic='testtopic', server='testserver', fanout=False)),
|
|
('fanout_target',
|
|
dict(topic='testtopic', server=None, fanout=True)),
|
|
]
|
|
|
|
_msg = [
|
|
('empty_msg',
|
|
dict(msg={}, expected={})),
|
|
('primitive_msg',
|
|
dict(msg={'foo': 'bar'}, expected={'foo': 'bar'})),
|
|
('complex_msg',
|
|
dict(msg={'a': {'b': datetime.datetime(1920, 2, 3, 4, 5, 6, 7)}},
|
|
expected={'a': {'b': '1920-02-03T04:05:06.000007'}})),
|
|
]
|
|
|
|
_context = [
|
|
('empty_ctxt', dict(ctxt={}, expected_ctxt={})),
|
|
('user_project_ctxt',
|
|
dict(ctxt={'user': 'mark', 'project': 'snarkybunch'},
|
|
expected_ctxt={'_context_user': 'mark',
|
|
'_context_project': 'snarkybunch'})),
|
|
]
|
|
|
|
@classmethod
|
|
def generate_scenarios(cls):
|
|
cls.scenarios = testscenarios.multiply_scenarios(cls._msg,
|
|
cls._context,
|
|
cls._target)
|
|
|
|
def setUp(self):
|
|
super(TestRequestWireFormat, self).setUp()
|
|
self.uuids = []
|
|
self.orig_uuid4 = uuid.uuid4
|
|
self.useFixture(fixtures.MonkeyPatch('uuid.uuid4', self.mock_uuid4))
|
|
|
|
def mock_uuid4(self):
|
|
self.uuids.append(self.orig_uuid4())
|
|
return self.uuids[-1]
|
|
|
|
def test_request_wire_format(self):
|
|
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
|
|
driver = transport._driver
|
|
|
|
target = oslo_messaging.Target(topic=self.topic,
|
|
server=self.server,
|
|
fanout=self.fanout)
|
|
|
|
connection, channel, queue = _declare_queue(target)
|
|
self.addCleanup(connection.release)
|
|
|
|
driver.send(target, self.ctxt, self.msg)
|
|
|
|
msgs = []
|
|
|
|
def callback(msg):
|
|
msg = channel.message_to_python(msg)
|
|
msg.ack()
|
|
msgs.append(msg.payload)
|
|
|
|
queue.consume(callback=callback,
|
|
consumer_tag='1',
|
|
nowait=False)
|
|
|
|
connection.drain_events()
|
|
|
|
self.assertEqual(1, len(msgs))
|
|
self.assertIn('oslo.message', msgs[0])
|
|
|
|
received = msgs[0]
|
|
received['oslo.message'] = jsonutils.loads(received['oslo.message'])
|
|
|
|
# FIXME(markmc): add _msg_id and _reply_q check
|
|
expected_msg = {
|
|
'_unique_id': self.uuids[0].hex,
|
|
}
|
|
expected_msg.update(self.expected)
|
|
expected_msg.update(self.expected_ctxt)
|
|
|
|
expected = {
|
|
'oslo.version': '2.0',
|
|
'oslo.message': expected_msg,
|
|
}
|
|
|
|
self.assertEqual(expected, received)
|
|
|
|
|
|
TestRequestWireFormat.generate_scenarios()
|
|
|
|
|
|
def _create_producer(target):
|
|
connection = kombu.connection.BrokerConnection(transport='memory')
|
|
|
|
# Kludge to speed up tests.
|
|
connection.transport.polling_interval = 0.0
|
|
|
|
connection.connect()
|
|
channel = connection.channel()
|
|
|
|
# work around 'memory' transport bug in 1.1.3
|
|
channel._new_queue('ae.undeliver')
|
|
|
|
if target.fanout:
|
|
exchange = kombu.entity.Exchange(name=target.topic + '_fanout',
|
|
type='fanout',
|
|
durable=False,
|
|
auto_delete=True)
|
|
producer = kombu.messaging.Producer(exchange=exchange,
|
|
channel=channel,
|
|
routing_key=target.topic)
|
|
elif target.server:
|
|
exchange = kombu.entity.Exchange(name='openstack',
|
|
type='topic',
|
|
durable=False,
|
|
auto_delete=False)
|
|
topic = '%s.%s' % (target.topic, target.server)
|
|
producer = kombu.messaging.Producer(exchange=exchange,
|
|
channel=channel,
|
|
routing_key=topic)
|
|
else:
|
|
exchange = kombu.entity.Exchange(name='openstack',
|
|
type='topic',
|
|
durable=False,
|
|
auto_delete=False)
|
|
producer = kombu.messaging.Producer(exchange=exchange,
|
|
channel=channel,
|
|
routing_key=target.topic)
|
|
|
|
return connection, producer
|
|
|
|
|
|
class TestReplyWireFormat(test_utils.BaseTestCase):
|
|
|
|
_target = [
|
|
('topic_target',
|
|
dict(topic='testtopic', server=None, fanout=False)),
|
|
('server_target',
|
|
dict(topic='testtopic', server='testserver', fanout=False)),
|
|
('fanout_target',
|
|
dict(topic='testtopic', server=None, fanout=True)),
|
|
]
|
|
|
|
_msg = [
|
|
('empty_msg',
|
|
dict(msg={}, expected={})),
|
|
('primitive_msg',
|
|
dict(msg={'foo': 'bar'}, expected={'foo': 'bar'})),
|
|
('complex_msg',
|
|
dict(msg={'a': {'b': '1920-02-03T04:05:06.000007'}},
|
|
expected={'a': {'b': '1920-02-03T04:05:06.000007'}})),
|
|
]
|
|
|
|
_context = [
|
|
('empty_ctxt', dict(ctxt={}, expected_ctxt={})),
|
|
('user_project_ctxt',
|
|
dict(ctxt={'_context_user': 'mark',
|
|
'_context_project': 'snarkybunch'},
|
|
expected_ctxt={'user': 'mark', 'project': 'snarkybunch'})),
|
|
]
|
|
|
|
@classmethod
|
|
def generate_scenarios(cls):
|
|
cls.scenarios = testscenarios.multiply_scenarios(cls._msg,
|
|
cls._context,
|
|
cls._target)
|
|
|
|
def test_reply_wire_format(self):
|
|
|
|
transport = oslo_messaging.get_transport(self.conf,
|
|
'kombu+memory:////')
|
|
self.addCleanup(transport.cleanup)
|
|
|
|
driver = transport._driver
|
|
|
|
target = oslo_messaging.Target(topic=self.topic,
|
|
server=self.server,
|
|
fanout=self.fanout)
|
|
|
|
listener = driver.listen(target)
|
|
|
|
connection, producer = _create_producer(target)
|
|
self.addCleanup(connection.release)
|
|
|
|
msg = {
|
|
'oslo.version': '2.0',
|
|
'oslo.message': {}
|
|
}
|
|
|
|
msg['oslo.message'].update(self.msg)
|
|
msg['oslo.message'].update(self.ctxt)
|
|
|
|
msg['oslo.message'].update({
|
|
'_msg_id': uuid.uuid4().hex,
|
|
'_unique_id': uuid.uuid4().hex,
|
|
'_reply_q': 'reply_' + uuid.uuid4().hex,
|
|
})
|
|
|
|
msg['oslo.message'] = jsonutils.dumps(msg['oslo.message'])
|
|
|
|
producer.publish(msg)
|
|
|
|
received = listener.poll()
|
|
self.assertIsNotNone(received)
|
|
self.assertEqual(self.expected_ctxt, received.ctxt)
|
|
self.assertEqual(self.expected, received.message)
|
|
|
|
|
|
TestReplyWireFormat.generate_scenarios()
|
|
|
|
|
|
class RpcKombuHATestCase(test_utils.BaseTestCase):
|
|
|
|
def setUp(self):
|
|
super(RpcKombuHATestCase, self).setUp()
|
|
self.brokers = ['host1', 'host2', 'host3', 'host4', 'host5']
|
|
self.config(rabbit_hosts=self.brokers,
|
|
rabbit_retry_interval=0.01,
|
|
rabbit_retry_backoff=0.01,
|
|
kombu_reconnect_delay=0,
|
|
heartbeat_timeout_threshold=0,
|
|
group="oslo_messaging_rabbit")
|
|
|
|
self.kombu_connect = mock.Mock()
|
|
self.useFixture(mockpatch.Patch(
|
|
'kombu.connection.Connection.connect',
|
|
side_effect=self.kombu_connect))
|
|
self.useFixture(mockpatch.Patch(
|
|
'kombu.connection.Connection.channel'))
|
|
|
|
# starting from the first broker in the list
|
|
url = oslo_messaging.TransportURL.parse(self.conf, None)
|
|
self.connection = rabbit_driver.Connection(self.conf, url,
|
|
amqp.PURPOSE_SEND)
|
|
self.addCleanup(self.connection.close)
|
|
|
|
def test_ensure_four_retry(self):
|
|
mock_callback = mock.Mock(side_effect=IOError)
|
|
self.assertRaises(oslo_messaging.MessageDeliveryFailure,
|
|
self.connection.ensure, mock_callback,
|
|
retry=4)
|
|
self.assertEqual(5, self.kombu_connect.call_count)
|
|
self.assertEqual(6, mock_callback.call_count)
|
|
|
|
def test_ensure_one_retry(self):
|
|
mock_callback = mock.Mock(side_effect=IOError)
|
|
self.assertRaises(oslo_messaging.MessageDeliveryFailure,
|
|
self.connection.ensure, mock_callback,
|
|
retry=1)
|
|
self.assertEqual(2, self.kombu_connect.call_count)
|
|
self.assertEqual(3, mock_callback.call_count)
|
|
|
|
def test_ensure_no_retry(self):
|
|
mock_callback = mock.Mock(side_effect=IOError)
|
|
self.assertRaises(oslo_messaging.MessageDeliveryFailure,
|
|
self.connection.ensure, mock_callback,
|
|
retry=0)
|
|
self.assertEqual(1, self.kombu_connect.call_count)
|
|
self.assertEqual(2, mock_callback.call_count)
|
|
|
|
|
|
class ConnectionLockTestCase(test_utils.BaseTestCase):
|
|
def _thread(self, lock, sleep, heartbeat=False):
|
|
def thread_task():
|
|
if heartbeat:
|
|
with lock.for_heartbeat():
|
|
time.sleep(sleep)
|
|
else:
|
|
with lock:
|
|
time.sleep(sleep)
|
|
|
|
t = threading.Thread(target=thread_task)
|
|
t.daemon = True
|
|
t.start()
|
|
start = time.time()
|
|
|
|
def get_elapsed_time():
|
|
t.join()
|
|
return time.time() - start
|
|
|
|
return get_elapsed_time
|
|
|
|
def test_workers_only(self):
|
|
l = rabbit_driver.ConnectionLock()
|
|
t1 = self._thread(l, 1)
|
|
t2 = self._thread(l, 1)
|
|
self.assertAlmostEqual(1, t1(), places=0)
|
|
self.assertAlmostEqual(2, t2(), places=0)
|
|
|
|
def test_worker_and_heartbeat(self):
|
|
l = rabbit_driver.ConnectionLock()
|
|
t1 = self._thread(l, 1)
|
|
t2 = self._thread(l, 1, heartbeat=True)
|
|
self.assertAlmostEqual(1, t1(), places=0)
|
|
self.assertAlmostEqual(2, t2(), places=0)
|
|
|
|
def test_workers_and_heartbeat(self):
|
|
l = rabbit_driver.ConnectionLock()
|
|
t1 = self._thread(l, 1)
|
|
t2 = self._thread(l, 1)
|
|
t3 = self._thread(l, 1)
|
|
t4 = self._thread(l, 1, heartbeat=True)
|
|
t5 = self._thread(l, 1)
|
|
self.assertAlmostEqual(1, t1(), places=0)
|
|
self.assertAlmostEqual(2, t4(), places=0)
|
|
self.assertAlmostEqual(3, t2(), places=0)
|
|
self.assertAlmostEqual(4, t3(), places=0)
|
|
self.assertAlmostEqual(5, t5(), places=0)
|
|
|
|
def test_heartbeat(self):
|
|
l = rabbit_driver.ConnectionLock()
|
|
t1 = self._thread(l, 1, heartbeat=True)
|
|
t2 = self._thread(l, 1)
|
|
self.assertAlmostEqual(1, t1(), places=0)
|
|
self.assertAlmostEqual(2, t2(), places=0)
|