[zmq] Reduce number of connections
In this change 'use_router_proxy' option was added
to switch between direct connections and proxy.
Proxy was reimplemented and splitted onto two
types of proxies:
* PUBLISHER proxy for fanout pattern
* ROUTER proxy for direct messaging
Each type of proxy is configured over command line
argument --type.
Deployment guide was updated accordingly to the change.
Change-Id: If36e9c26e2a8ebe622cfa7e9f2a07b1a69aabe34
Closes-Bug: #1555007
changes/94/287094/20
Oleksii Zamiatin
7 years ago
committed by
ozamiatin
parent
3728ccc831
commit
cc1cb30321
@ -1,42 +0,0 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from oslo_messaging._drivers import impl_zmq
|
||||
from oslo_messaging._drivers.zmq_driver.broker import zmq_broker
|
||||
from oslo_messaging import server
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(impl_zmq.zmq_opts)
|
||||
CONF.register_opts(server._pool_opts)
|
||||
CONF.rpc_zmq_native = True
|
||||
|
||||
|
||||
def main():
|
||||
CONF(sys.argv[1:], project='oslo')
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
reactor = zmq_broker.ZmqBroker(CONF)
|
||||
reactor.start()
|
||||
|
||||
while True:
|
||||
time.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,74 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from oslo_messaging._drivers import impl_zmq
|
||||
from oslo_messaging._drivers.zmq_driver.broker import zmq_proxy
|
||||
from oslo_messaging import server
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(impl_zmq.zmq_opts)
|
||||
CONF.register_opts(server._pool_opts)
|
||||
CONF.rpc_zmq_native = True
|
||||
|
||||
|
||||
USAGE = """ Usage: ./zmq-proxy.py --type {PUBLISHER,ROUTER} [-h] [] ...
|
||||
|
||||
Usage example:
|
||||
python oslo_messaging/_cmd/zmq-proxy.py\
|
||||
--type PUBLISHER"""
|
||||
|
||||
|
||||
PUBLISHER = 'PUBLISHER'
|
||||
ROUTER = 'ROUTER'
|
||||
PROXY_TYPES = (PUBLISHER, ROUTER)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description='ZeroMQ proxy service',
|
||||
usage=USAGE
|
||||
)
|
||||
|
||||
parser.add_argument('--type', dest='proxy_type', type=str,
|
||||
default=PUBLISHER,
|
||||
help='Proxy type PUBLISHER or ROUTER')
|
||||
parser.add_argument('--config-file', dest='config_file', type=str,
|
||||
help='Path to configuration file')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.config_file:
|
||||
cfg.CONF(["--config-file", args.config_file])
|
||||
|
||||
if args.proxy_type not in PROXY_TYPES:
|
||||
raise Exception("Bad proxy type %s, should be one of %s" %
|
||||
(args.proxy_type, PROXY_TYPES))
|
||||
|
||||
reactor = zmq_proxy.ZmqPublisher(CONF) if args.proxy_type == PUBLISHER \
|
||||
else zmq_proxy.ZmqRouter(CONF)
|
||||
|
||||
reactor.start()
|
||||
|
||||
while True:
|
||||
time.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,78 +0,0 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from oslo_utils import excutils
|
||||
from stevedore import driver
|
||||
|
||||
from oslo_messaging._drivers.zmq_driver.broker import zmq_queue_proxy
|
||||
from oslo_messaging._drivers.zmq_driver import zmq_async
|
||||
from oslo_messaging._i18n import _LE, _LI
|
||||
|
||||
zmq = zmq_async.import_zmq(zmq_concurrency='native')
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ZmqBroker(object):
|
||||
"""Local messaging IPC broker (nodes are still peers).
|
||||
The main purpose is to have native zeromq application.
|
||||
Benefits of such approach are following:
|
||||
|
||||
1. No risk to block the main thread of the process by unpatched
|
||||
native parts of the libzmq (c-library is completely monkey-patch
|
||||
unfriendly)
|
||||
2. Making use of standard zmq approaches as async pollers,
|
||||
devices, queues etc.
|
||||
3. Possibility to implement queue persistence not touching existing
|
||||
clients (staying in a separate process).
|
||||
"""
|
||||
|
||||
def __init__(self, conf):
|
||||
super(ZmqBroker, self).__init__()
|
||||
self.conf = conf
|
||||
self._create_ipc_dirs()
|
||||
self.matchmaker = driver.DriverManager(
|
||||
'oslo.messaging.zmq.matchmaker',
|
||||
self.conf.rpc_zmq_matchmaker,
|
||||
).driver(self.conf)
|
||||
|
||||
self.context = zmq.Context()
|
||||
self.proxies = [zmq_queue_proxy.UniversalQueueProxy(
|
||||
conf, self.context, self.matchmaker)
|
||||
]
|
||||
|
||||
def _create_ipc_dirs(self):
|
||||
ipc_dir = self.conf.rpc_zmq_ipc_dir
|
||||
try:
|
||||
os.makedirs("%s/fanout" % ipc_dir)
|
||||
except os.error:
|
||||
if not os.path.isdir(ipc_dir):
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.error(_LE("Required IPC directory does not exist at"
|
||||
" %s"), ipc_dir)
|
||||
|
||||
def start(self):
|
||||
for proxy in self.proxies:
|
||||
proxy.start()
|
||||
|
||||
def wait(self):
|
||||
for proxy in self.proxies:
|
||||
proxy.wait()
|
||||
|
||||
def close(self):
|
||||
LOG.info(_LI("Broker shutting down ..."))
|
||||
for proxy in self.proxies:
|
||||
proxy.stop()
|
||||
@ -0,0 +1,115 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from oslo_utils import excutils
|
||||
from stevedore import driver
|
||||
|
||||
from oslo_messaging._drivers.zmq_driver.broker import zmq_queue_proxy
|
||||
from oslo_messaging._drivers.zmq_driver import zmq_async
|
||||
from oslo_messaging._i18n import _LE, _LI
|
||||
|
||||
zmq = zmq_async.import_zmq(zmq_concurrency='native')
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ZmqProxy(object):
|
||||
"""Base class for Publishers and Routers proxies.
|
||||
The main reason to have a proxy is high complexity of TCP sockets number
|
||||
growth with direct connections (when services connect directly to
|
||||
each other). The general complexity for ZeroMQ+Openstack deployment
|
||||
with direct connections may be square(N) (where N is a number of nodes
|
||||
in deployment). With proxy the complexity is reduced to k*N where
|
||||
k is a number of services.
|
||||
|
||||
Currently there are 2 types of proxy, they are Publishers and Routers.
|
||||
Publisher proxy serves for PUB-SUB pattern implementation where
|
||||
Publisher is a server which performs broadcast to subscribers.
|
||||
Router is used for direct message types in case of number of TCP socket
|
||||
connections is critical for specific deployment. Generally 3 publishers
|
||||
is enough for deployment. Routers should be
|
||||
"""
|
||||
|
||||
def __init__(self, conf):
|
||||
super(ZmqProxy, self).__init__()
|
||||
self.conf = conf
|
||||
self._create_ipc_dirs()
|
||||
self.matchmaker = driver.DriverManager(
|
||||
'oslo.messaging.zmq.matchmaker',
|
||||
self.conf.rpc_zmq_matchmaker,
|
||||
).driver(self.conf)
|
||||
self.context = zmq.Context()
|
||||
self.proxies = []
|
||||
|
||||
def _create_ipc_dirs(self):
|
||||
ipc_dir = self.conf.rpc_zmq_ipc_dir
|
||||
try:
|
||||
os.makedirs("%s/fanout" % ipc_dir)
|
||||
except os.error:
|
||||
if not os.path.isdir(ipc_dir):
|
||||
with excutils.save_and_reraise_exception():
|
||||
LOG.error(_LE("Required IPC directory does not exist at"
|
||||
" %s"), ipc_dir)
|
||||
|
||||
def start(self):
|
||||
for proxy in self.proxies:
|
||||
proxy.start()
|
||||
|
||||
def wait(self):
|
||||
for proxy in self.proxies:
|
||||
proxy.wait()
|
||||
|
||||
def close(self):
|
||||
LOG.info(_LI("Broker shutting down ..."))
|
||||
for proxy in self.proxies:
|
||||
proxy.stop()
|
||||
|
||||
|
||||
class ZmqPublisher(ZmqProxy):
|
||||
|
||||
def __init__(self, conf):
|
||||
super(ZmqPublisher, self).__init__(conf)
|
||||
self.proxies.append(zmq_queue_proxy.PublisherProxy(
|
||||
conf, self.context, self.matchmaker))
|
||||
|
||||
|
||||
class ZmqRouter(ZmqProxy):
|
||||
"""Router is used for direct messages in order to reduce the number of
|
||||
allocated TCP sockets in controller. The list of requirements to Router:
|
||||
|
||||
1. There may be any number of routers in the deployment. Routers are
|
||||
registered in a name-server and client connects dynamically to all of
|
||||
them performing load balancing.
|
||||
2. Routers should be transparent for clients and servers. Which means
|
||||
it doesn't change the way of messaging between client and the final
|
||||
target by hiding the target from a client.
|
||||
3. Router may be restarted or get down at any time loosing all messages
|
||||
in its queue. Smart retrying (based on acknowledgements from server
|
||||
side) and load balancing between other Router instances from the
|
||||
client side should handle the situation.
|
||||
4. Router takes all the routing information from message envelope and
|
||||
doesn't perform Target-resolution in any way.
|
||||
5. Routers don't talk to each other and no synchronization is needed.
|
||||
6. Load balancing is performed by the client in a round-robin fashion.
|
||||
|
||||
Those requirements should limit the performance impact caused by using
|
||||
of proxies making proxies as lightweight as possible.
|
||||
"""
|
||||
|
||||
def __init__(self, conf):
|
||||
super(ZmqRouter, self).__init__(conf)
|
||||
self.proxies.append(zmq_queue_proxy.RouterProxy(
|
||||
conf, self.context, self.matchmaker))
|
||||
@ -0,0 +1,67 @@
|
||||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from oslo_messaging._drivers.zmq_driver.client.publishers \
|
||||
import zmq_publisher_base
|
||||
from oslo_messaging._drivers.zmq_driver import zmq_async
|
||||
from oslo_messaging._drivers.zmq_driver import zmq_names
|
||||
|
||||
zmq = zmq_async.import_zmq()
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DealerPublisherProxy(object):
|
||||
|
||||
def __init__(self, conf, matchmaker, poller):
|
||||
super(DealerPublisherProxy, self).__init__()
|
||||
self.conf = conf
|
||||
self.matchmaker = matchmaker
|
||||
self.poller = poller
|
||||
self.sockets_manager = zmq_publisher_base.SocketsManager(
|
||||
conf, matchmaker, zmq.ROUTER, zmq.DEALER)
|
||||
|
||||
def send_request(self, multipart_message):
|
||||
envelope = multipart_message[zmq_names.MULTIPART_IDX_ENVELOPE]
|
||||
if envelope.is_mult_send:
|
||||
raise zmq_publisher_base.UnsupportedSendPattern(envelope.msg_type)
|
||||
if not envelope.target_hosts:
|
||||
raise Exception("Target hosts are expected!")
|
||||
|
||||
dealer_socket = self.sockets_manager.get_socket_to_hosts(
|
||||
envelope.target, envelope.target_hosts)
|
||||
self.poller.register(dealer_socket.handle, self.receive_reply)
|
||||
|
||||
LOG.debug("Sending message %(message)s to a target %(target)s"
|
||||
% {"message": envelope.message_id,
|
||||
"target": envelope.target})
|
||||
|
||||
# Empty delimiter - DEALER socket specific
|
||||
dealer_socket.send(b'', zmq.SNDMORE)
|
||||
dealer_socket.send_pyobj(envelope, zmq.SNDMORE)
|
||||
dealer_socket.send(multipart_message[zmq_names.MULTIPART_IDX_BODY])
|
||||
|
||||
def receive_reply(self, socket):
|
||||
empty = socket.recv()
|
||||
assert empty == b'', "Empty expected!"
|
||||
envelope = socket.recv_pyobj()
|
||||
assert envelope is not None, "Invalid envelope!"
|
||||
reply = socket.recv()
|
||||
LOG.debug("Received reply %s", reply)
|
||||
return [envelope, reply]
|
||||
|
||||
def cleanup(self):
|
||||
self.sockets_manager.cleanup()
|
||||
Loading…
Reference in New Issue