Ensure API workers > 1 starts correctly

* Move DNSUtils bind methods into designate.utils
* Switch WSGIService to using the bind_tcp method
* Add support for TCP_KEEPIDLE to the bind_tcp method

Change-Id: If2d554fd757969f5b833ede6cb635bb4901906a7
Closes-Bug: 1498054

designate/dnsutils.py

@@ -323,46 +323,6 @@ def dnspythonrecord_to_recordset(rname, rdataset):
     return rrset
 
 
-def bind_tcp(host, port, tcp_backlog):
-    # Bind to the TCP port
-    LOG.info(_LI('Opening TCP Listening Socket on %(host)s:%(port)d') %
-             {'host': host, 'port': port})
-    sock_tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    sock_tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    sock_tcp.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-
-    # NOTE: Linux supports socket.SO_REUSEPORT only in 3.9 and later releases.
-    try:
-        sock_tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
-    except Exception:
-        pass
-
-    sock_tcp.setblocking(True)
-    sock_tcp.bind((host, port))
-    sock_tcp.listen(tcp_backlog)
-
-    return sock_tcp
-
-
-def bind_udp(host, port):
-    # Bind to the UDP port
-    LOG.info(_LI('Opening UDP Listening Socket on %(host)s:%(port)d') %
-             {'host': host, 'port': port})
-    sock_udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-    sock_udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-
-    # NOTE: Linux supports socket.SO_REUSEPORT only in 3.9 and later releases.
-    try:
-        sock_udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
-    except Exception:
-        pass
-
-    sock_udp.setblocking(True)
-    sock_udp.bind((host, port))
-
-    return sock_udp
-
-
 def do_axfr(zone_name, servers, timeout=None, source=None):
     """
     Performs an AXFR for a given zone name

designate/service.py

@@ -21,7 +21,6 @@ import abc
 import socket
 import struct
 import errno
-import time
 
 import six
 import eventlet.wsgi
@@ -40,7 +39,7 @@ from designate.i18n import _LW
 from designate import rpc
 from designate import policy
 from designate import version
-from designate import dnsutils
+from designate import utils
 
 # TODO(kiall): These options have been cut+paste from the old WSGI code, and
 #              should be moved into service:api etc..
@@ -171,62 +170,24 @@ class WSGIService(object):
     def start(self):
         super(WSGIService, self).start()
 
-        socket = self._wsgi_get_socket()
+        self._wsgi_sock = utils.bind_tcp(
-        application = self._wsgi_application
+            self._service_config.api_host,
+            self._service_config.api_port,
+            CONF.backlog,
+            CONF.tcp_keepidle)
 
-        self.tg.add_thread(self._wsgi_handle, application, socket)
+        if sslutils.is_enabled(CONF):
+            self._wsgi_sock = sslutils.wrap(CONF, self._wsgi_sock)
 
-    def _wsgi_get_socket(self):
+        self.tg.add_thread(self._wsgi_handle)
-        # TODO(dims): eventlet's green dns/socket module does not actually
-        # support IPv6 in getaddrinfo(). We need to get around this in the
-        # future or monitor upstream for a fix
-        info = socket.getaddrinfo(self._service_config.api_host,
-                                  self._service_config.api_port,
-                                  socket.AF_UNSPEC,
-                                  socket.SOCK_STREAM)[0]
-        family = info[0]
-        bind_addr = info[-1]
 
-        sock = None
+    def _wsgi_handle(self):
-        retry_until = time.time() + 30
-        while not sock and time.time() < retry_until:
-            try:
-                # TODO(kiall): Backlog should be a service specific setting,
-                #              rather than a global
-                sock = eventlet.listen(bind_addr,
-                                       backlog=cfg.CONF.backlog,
-                                       family=family)
-                if sslutils.is_enabled(CONF):
-                    sock = sslutils.wrap(CONF, sock)
-
-            except socket.error as err:
-                if err.args[0] != errno.EADDRINUSE:
-                    raise
-                eventlet.sleep(0.1)
-        if not sock:
-            raise RuntimeError(_("Could not bind to %(host)s:%(port)s "
-                               "after trying for 30 seconds") %
-                               {'host': self._service_config.api_host,
-                                'port': self._service_config.api_port})
-        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-        # sockets can hang around forever without keepalive
-        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-
-        # This option isn't available in the OS X version of eventlet
-        if hasattr(socket, 'TCP_KEEPIDLE'):
-            sock.setsockopt(socket.IPPROTO_TCP,
-                            socket.TCP_KEEPIDLE,
-                            CONF.tcp_keepidle)
-
-        return sock
-
-    def _wsgi_handle(self, application, socket):
         logger = logging.getLogger('eventlet.wsgi')
         # Adjust wsgi MAX_HEADER_LINE to accept large tokens.
         eventlet.wsgi.MAX_HEADER_LINE = self._service_config.max_header_line
 
-        eventlet.wsgi.server(socket,
+        eventlet.wsgi.server(self._wsgi_sock,
-                             application,
+                             self._wsgi_application,
                              custom_pool=self.tg.pool,
                              log=loggers.WritableLogger(logger))
 
@@ -250,12 +211,12 @@ class DNSService(object):
     def start(self):
         super(DNSService, self).start()
 
-        self._dns_sock_tcp = dnsutils.bind_tcp(
+        self._dns_sock_tcp = utils.bind_tcp(
             self._service_config.host,
             self._service_config.port,
             self._service_config.tcp_backlog)
 
-        self._dns_sock_udp = dnsutils.bind_udp(
+        self._dns_sock_udp = utils.bind_udp(
             self._service_config.host,
             self._service_config.port)
 

designate/utils.py

@@ -19,6 +19,7 @@ import functools
 import inspect
 import os
 import uuid
+import socket
 
 import six
 import pkg_resources
@@ -30,6 +31,7 @@ from oslo_utils import timeutils
 
 from designate import exceptions
 from designate.i18n import _
+from designate.i18n import _LI
 from designate.openstack.common.report import guru_meditation_report as gmr
 from designate import version as designate_version
 
@@ -453,3 +455,49 @@ def get_paging_params(params, sort_keys):
         raise exceptions.InvalidSortKey(msg)
 
     return marker, limit, sort_key, sort_dir
+
+
+def bind_tcp(host, port, tcp_backlog, tcp_keepidle=None):
+    # Bind to the TCP port
+    LOG.info(_LI('Opening TCP Listening Socket on %(host)s:%(port)d') %
+             {'host': host, 'port': port})
+    sock_tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    sock_tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    sock_tcp.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+
+    # NOTE: Linux supports socket.SO_REUSEPORT only in 3.9 and later releases.
+    try:
+        sock_tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+    except Exception:
+        pass
+
+    # This option isn't available in the OS X version of eventlet
+    if tcp_keepidle and hasattr(socket, 'TCP_KEEPIDLE'):
+        sock_tcp.setsockopt(socket.IPPROTO_TCP,
+                            socket.TCP_KEEPIDLE,
+                            tcp_keepidle)
+
+    sock_tcp.setblocking(True)
+    sock_tcp.bind((host, port))
+    sock_tcp.listen(tcp_backlog)
+
+    return sock_tcp
+
+
+def bind_udp(host, port):
+    # Bind to the UDP port
+    LOG.info(_LI('Opening UDP Listening Socket on %(host)s:%(port)d') %
+             {'host': host, 'port': port})
+    sock_udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    sock_udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+    # NOTE: Linux supports socket.SO_REUSEPORT only in 3.9 and later releases.
+    try:
+        sock_udp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+    except Exception:
+        pass
+
+    sock_udp.setblocking(True)
+    sock_udp.bind((host, port))
+
+    return sock_udp