Improve usability when backdoor_port is nonzero

Users who may not know that configuring a backdoor_port with 0
allows multiple services to be enabled for the eventlet backdoor
or who simply want a more predictable port assignment might like
this patch. If the specified port is in use, it is incremented
until a free port is found. This is a backdoor_port collision
recovery scheme as opposed to the collision failure scheme
that exists today.

This related to I95fdb5ca: Add support for backdoor_port to be
returned with a rpc call.

Change-Id: I7ec346db3575995fa15483b617eea34c1e003bb0

openstack/common/eventlet_backdoor.py

@@ -18,8 +18,11 @@
 
 from __future__ import print_function
 
+import errno
 import gc
+import os
 import pprint
+import socket
 import sys
 import traceback
 
@@ -28,14 +31,34 @@ import eventlet.backdoor
 import greenlet
 from oslo.config import cfg
 
+from openstack.common.gettextutils import _
+from openstack.common import log as logging
+
+help_for_backdoor_port = 'Acceptable ' + \
+    'values are 0, <port> and <start>:<end>, where 0 results in ' + \
+    'listening on a random tcp port number, <port> results in ' + \
+    'listening on the specified port number and not enabling backdoor' + \
+    'if it is in use and <start>:<end> results in listening on the ' + \
+    'smallest unused port number within the specified range of port ' + \
+    'numbers. The chosen port is displayed in the service\'s log file.'
 eventlet_backdoor_opts = [
-    cfg.IntOpt('backdoor_port',
+    cfg.StrOpt('backdoor_port',
                default=None,
-               help='port for eventlet backdoor to listen')
+               help='Enable eventlet backdoor. %s' % help_for_backdoor_port)
 ]
 
 CONF = cfg.CONF
 CONF.register_opts(eventlet_backdoor_opts)
+LOG = logging.getLogger(__name__)
+
+
+class EventletBackdoorConfigValueError(Exception):
+    def __init__(self, port_range, help_msg, ex):
+        msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '
+               '%(help)s' %
+               {'range': port_range, 'ex': ex, 'help': help_msg})
+        super(EventletBackdoorConfigValueError, self).__init__(msg)
+        self.port_range = port_range
 
 
 def _dont_use_this():
@@ -60,6 +83,33 @@ def _print_nativethreads():
         print()
 
 
+def _parse_port_range(port_range):
+    if ':' not in port_range:
+        start, end = port_range, port_range
+    else:
+        start, end = port_range.split(':', 1)
+    try:
+        start, end = int(start), int(end)
+        if end < start:
+            raise ValueError
+        return start, end
+    except ValueError as ex:
+        raise EventletBackdoorConfigValueError(port_range, ex,
+                                               help_for_backdoor_port)
+
+
+def _listen(host, start_port, end_port, listen_func):
+    try_port = start_port
+    while True:
+        try:
+            return listen_func((host, try_port))
+        except socket.error as exc:
+            if (exc.errno != errno.EADDRINUSE or
+               try_port >= end_port):
+                raise
+            try_port += 1
+
+
 def initialize_if_enabled():
     backdoor_locals = {
         'exit': _dont_use_this,      # So we don't exit the entire process
@@ -72,6 +122,8 @@ def initialize_if_enabled():
     if CONF.backdoor_port is None:
         return None
 
+    start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
+
     # NOTE(johannes): The standard sys.displayhook will print the value of
     # the last expression and set it to __builtin__._, which overwrites
     # the __builtin__._ that gettext sets. Let's switch to using pprint
@@ -82,8 +134,13 @@ def initialize_if_enabled():
             pprint.pprint(val)
     sys.displayhook = displayhook
 
-    sock = eventlet.listen(('localhost', CONF.backdoor_port))
+    sock = _listen('localhost', start_port, end_port, eventlet.listen)
+
+    # In the case of backdoor port being zero, a port number is assigned by
+    # listen().  In any case, pull the port number out here.
     port = sock.getsockname()[1]
+    LOG.info(_('Eventlet backdoor listening on %(port)s for process %(pid)d') %
+             {'port': port, 'pid': os.getpid()})
     eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
                      locals=backdoor_locals)
     return port

tests/unit/test_eventlet_backdoor.py

@@ -0,0 +1,71 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Unit Tests for eventlet backdoor
+"""
+import errno
+import eventlet
+import mox
+import socket
+
+from openstack.common import eventlet_backdoor
+from tests import utils
+
+
+class BackdoorPortTest(utils.BaseTestCase):
+
+    def common_backdoor_port_setup(self):
+        self.sock = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(eventlet, 'listen')
+        self.mox.StubOutWithMock(eventlet, 'spawn_n')
+
+    def test_backdoor_port_inuse(self):
+        self.config(backdoor_port=2345)
+        self.common_backdoor_port_setup()
+        eventlet.listen(('localhost', 2345)).AndRaise(
+            socket.error(errno.EADDRINUSE, ''))
+        self.mox.ReplayAll()
+        self.assertRaises(socket.error,
+                          eventlet_backdoor.initialize_if_enabled)
+
+    def test_backdoor_port_range(self):
+        self.config(backdoor_port='8800:8899')
+        self.common_backdoor_port_setup()
+        eventlet.listen(('localhost', 8800)).AndReturn(self.sock)
+        self.sock.getsockname().AndReturn(('127.0.0.1', 8800))
+        eventlet.spawn_n(eventlet.backdoor.backdoor_server, self.sock,
+                         locals=mox.IsA(dict))
+        self.mox.ReplayAll()
+        port = eventlet_backdoor.initialize_if_enabled()
+        self.assertEqual(port, 8800)
+
+    def test_backdoor_port_range_all_inuse(self):
+        self.config(backdoor_port='8800:8899')
+        self.common_backdoor_port_setup()
+        for i in range(8800, 8900):
+            eventlet.listen(('localhost', i)).AndRaise(
+                socket.error(errno.EADDRINUSE, ''))
+        self.mox.ReplayAll()
+        self.assertRaises(socket.error,
+                          eventlet_backdoor.initialize_if_enabled)
+
+    def test_backdoor_port_bad(self):
+        self.config(backdoor_port='abc')
+        self.assertRaises(eventlet_backdoor.EventletBackdoorConfigValueError,
+                          eventlet_backdoor.initialize_if_enabled)

tests/unit/test_service.py

@@ -17,18 +17,23 @@
 #    under the License.
 
 """
-Unit Tests for remote procedure calls using queue
+Unit Tests for service class
 """
 
 from __future__ import print_function
 
+import errno
+import eventlet
+import mox
 import os
 import signal
+import socket
 import time
 import traceback
 
 from oslo.config import cfg
 
+from openstack.common import eventlet_backdoor
 from openstack.common import log as logging
 from openstack.common.notifier import api as notifier_api
 from openstack.common import service
@@ -191,10 +196,59 @@ class ServiceLauncherTest(utils.BaseTestCase):
 
 
 class LauncherTest(utils.BaseTestCase):
+
     def test_backdoor_port(self):
-        # backdoor port should get passed to the service being launched
-        self.config(backdoor_port=1234)
+        self.config(backdoor_port='1234')
+
+        sock = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(eventlet, 'listen')
+        self.mox.StubOutWithMock(eventlet, 'spawn_n')
+
+        eventlet.listen(('localhost', 1234)).AndReturn(sock)
+        sock.getsockname().AndReturn(('127.0.0.1', 1234))
+        eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
+                         locals=mox.IsA(dict))
+
+        self.mox.ReplayAll()
+
         svc = service.Service()
         launcher = service.launch(svc)
-        self.assertEqual(1234, svc.backdoor_port)
+        self.assertEqual(svc.backdoor_port, 1234)
         launcher.stop()
+
+    def test_backdoor_inuse(self):
+        sock = eventlet.listen(('localhost', 0))
+        port = sock.getsockname()[1]
+        self.config(backdoor_port=port)
+        svc = service.Service()
+        self.assertRaises(socket.error,
+                          service.launch, svc)
+        sock.close()
+
+    def test_backdoor_port_range_one_inuse(self):
+        self.config(backdoor_port='8800:8900')
+
+        sock = self.mox.CreateMockAnything()
+        self.mox.StubOutWithMock(eventlet, 'listen')
+        self.mox.StubOutWithMock(eventlet, 'spawn_n')
+
+        eventlet.listen(('localhost', 8800)).AndRaise(
+            socket.error(errno.EADDRINUSE, ''))
+        eventlet.listen(('localhost', 8801)).AndReturn(sock)
+        sock.getsockname().AndReturn(('127.0.0.1', 8801))
+        eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
+                         locals=mox.IsA(dict))
+
+        self.mox.ReplayAll()
+
+        svc = service.Service()
+        launcher = service.launch(svc)
+        self.assertEqual(svc.backdoor_port, 8801)
+        launcher.stop()
+
+    def test_backdoor_port_reverse_range(self):
+        # backdoor port should get passed to the service being launched
+        self.config(backdoor_port='8888:7777')
+        svc = service.Service()
+        self.assertRaises(eventlet_backdoor.EventletBackdoorConfigValueError,
+                          service.launch, svc)