Merge "Remove eventlet from requirements and clean up references"

.coveragerc

@@ -2,7 +2,7 @@
 branch = True
 source = designate
 omit = designate/tests/*,designate/hacking/*
-concurrency = greenlet
+concurrency = thread
 
 [report]
 ignore_errors = True

designate/__init__.py

@@ -15,8 +15,10 @@
 # under the License.
 import os
 
-# Eventlet's GreenDNS Patching will prevent the resolution of names in
-# the /etc/hosts file, causing problems for installs.
+# Disable eventlet's greendns monkey patching to prevent dnspython
+# compatibility issues. Without this, dnspython's zone parsing fails with
+# errors like "TypeError: add(): expected an Rdata" due to conflicts between
+# eventlet's patched DNS resolver and dnspython's native implementation.
 os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 
 from oslo_concurrency import lockutils  # noqa

designate/api/wsgi.py

@@ -10,32 +10,23 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 """WSGI script for Designate API."""
-
 import os
 
-# NOTE(oschwart): remove once the default backend is ``BackendType.THREADING``
-import oslo_service.backend as service
-try:
-    service.init_backend(service.BackendType.THREADING)
-except service.exceptions.BackendAlreadySelected:
-    pass
+from oslo_config import cfg
+from oslo_log import log as logging
+import oslo_messaging as messaging
+from paste import deploy
 
-import oslo_messaging as messaging  # noqa: E402
+from designate.common import config
+from designate.common import profiler
+import designate.conf
+from designate import heartbeat_emitter
+from designate import policy
+from designate import rpc
 
 # Set some Oslo RPC defaults
 messaging.set_transport_defaults('designate')
 
-from oslo_config import cfg  # noqa: E402
-from oslo_log import log as logging  # noqa: E402
-from paste import deploy  # noqa: E402
-
-from designate.common import config  # noqa: E402
-from designate.common import profiler  # noqa: E402
-import designate.conf  # noqa: E402
-from designate import heartbeat_emitter  # noqa: E402
-from designate import policy  # noqa: E402
-from designate import rpc  # noqa: E402
-
 CONF = designate.conf.CONF
 
 CONFIG_FILES = ['api-paste.ini', 'designate.conf']

designate/central/service.py

@@ -21,7 +21,6 @@ import random
 from random import SystemRandom
 import re
 import string
-import time
 
 from dns import exception as dnsexception
 from dns import zone as dnszone
@@ -832,9 +831,6 @@ class Service(service.RPCService):
 
         if zone.obj_attr_is_set('recordsets'):
             for rrset in zone.recordsets:
-                # This allows eventlet to yield, as this looping operation
-                # can be very long-lived.
-                time.sleep(0)
                 self._create_recordset_in_storage(
                     context, zone, rrset, increment_serial=False
                 )

designate/cmd/__init__.py

@@ -0,0 +1,45 @@
+# Copyright 2012 Managed I.T.
+#
+# Author: Kiall Mac Innes <kiall@managedit.ie>
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from oslo_service import backend
+from oslo_service.backend import exceptions as backend_exceptions
+
+# Only initialize backend if not already set
+try:
+    backend.init_backend(backend.BackendType.THREADING)
+except backend_exceptions.BackendAlreadySelected:
+    # Backend already initialized, this is fine
+    pass
+
+from oslo_log import log  # noqa
+from oslo_concurrency import lockutils  # noqa
+import oslo_messaging as messaging  # noqa
+
+_EXTRA_DEFAULT_LOG_LEVELS = [
+    'kazoo.client=WARN',
+    'keystone=INFO',
+    'oslo_service.loopingcall=WARN',
+]
+
+# Set some Oslo Log defaults
+log.set_defaults(default_log_levels=log.get_default_log_levels() +
+                 _EXTRA_DEFAULT_LOG_LEVELS)
+
+# Set some Oslo RPC defaults
+messaging.set_transport_defaults('designate')
+
+# Set some Oslo Concurrency defaults
+lockutils.set_defaults(lock_path='$state_path')

designate/cmd/threading/__init__.py

@@ -1,27 +0,0 @@
-# Copyright 2012 Managed I.T.
-#
-# Author: Kiall Mac Innes <kiall@managedit.ie>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# NOTE(oschwart): remove once the default backend is ``BackendType.THREADING``
-import oslo_service.backend as service
-try:
-    service.init_backend(service.BackendType.THREADING)
-except service.exceptions.BackendAlreadySelected:
-    pass
-
-import oslo_messaging as messaging  # noqa
-
-# Set some Oslo RPC defaults
-messaging.set_transport_defaults('designate')

designate/conf/sink.py

@@ -39,7 +39,7 @@ SINK_OPTS = [
     cfg.IntOpt('workers',
                help='Number of sink worker processes to spawn'),
     cfg.IntOpt('threads', default=1000,
-               help='Number of sink greenthreads to spawn'),
+               help='Number of sink threads to spawn'),
     cfg.ListOpt('enabled_notification_handlers', default=[],
                 help='Enabled Notification Handlers'),
     cfg.StrOpt('listener_pool_name',

designate/service.py

@@ -273,7 +273,7 @@ class DNSService:
         through the same TCP connection but they will be processed
         sequentially.
         See https://tools.ietf.org/html/draft-ietf-dnsop-5966bis-03
-        Raises no exception: it's to be run in an eventlet green thread
+        Raises no exception: it's to be run in a thread
 
         :param addr: Tuple of the client's (IPv4 addr, Port) or
                      (IPv6 addr, Port, Flow info, Scope ID)

designate/tests/__init__.py

@@ -14,11 +14,18 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-
 import os
+
+# Disable eventlet's greendns monkey patching to prevent dnspython
+# compatibility issues. Without this, dnspython's zone parsing fails with
+# errors like "TypeError: add(): expected an Rdata" due to conflicts between
+# eventlet's patched DNS resolver and dnspython's native implementation.
 os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
 
-import eventlet # noqa
-
-
-eventlet.monkey_patch(os=False) # noqa
+from oslo_service import backend as oslo_service_backend # noqa
+try:
+    oslo_service_backend.init_backend(
+        oslo_service_backend.BackendType.THREADING)
+except oslo_service_backend.exceptions.BackendAlreadySelected:
+    # Backend already initialized, this is fine
+    pass

designate/tests/functional/api/test_service.py

@@ -29,6 +29,12 @@ class ApiServiceTest(designate.tests.functional.TestCase):
 
         self.config(listen=['0.0.0.0:0'], group='service:api')
 
+        # Mock oslo_service.wsgi.Server since it's incompatible with threading
+        # backend
+        self.wsgi_server_patcher = mock.patch('oslo_service.wsgi.Server')
+        self.mock_wsgi_server = self.wsgi_server_patcher.start()
+        self.addCleanup(self.wsgi_server_patcher.stop)
+
         self.service = service.Service()
 
     def test_start_and_stop(self):

designate/tests/functional/test_upgrade_checks.py

@@ -17,7 +17,7 @@ from oslo_upgradecheck import upgradecheck
 from sqlalchemy.schema import MetaData
 from sqlalchemy.schema import Table
 
-from designate.cmd.threading import status
+from designate.cmd import status
 from designate.storage import sql
 import designate.tests.functional
 

designate/tests/unit/cmd/test_cmd.py

@@ -14,21 +14,18 @@ from unittest import mock
 from oslo_config import fixture as cfg_fixture
 import oslotest.base
 
+from designate.cmd import api
+from designate.cmd import central
+from designate.cmd import mdns
+from designate.cmd import producer
+from designate.cmd import sink
+from designate.cmd import worker
 import designate.conf
 
 
 CONF = designate.conf.CONF
 
 
-with mock.patch('oslo_service.backend.init_backend'):
-    from designate.cmd.threading import api
-    from designate.cmd.threading import central
-    from designate.cmd.threading import mdns
-    from designate.cmd.threading import producer
-    from designate.cmd.threading import sink
-    from designate.cmd.threading import worker
-
-
 @mock.patch('designate.service.wait')
 @mock.patch('designate.service.serve')
 @mock.patch('designate.heartbeat_emitter.get_heartbeat_emitter')
@@ -122,3 +119,108 @@ class CmdTestCase(oslotest.base.BaseTestCase):
         mock_heartbeat.assert_called()
         mock_serve.assert_called_with(mock.ANY, workers=1)
         mock_wait.assert_called_with()
+
+    @mock.patch('designate.api.service.Service')
+    def test_api_rpc_already_initialized(self, mock_service, mock_read_config,
+                                         mock_log_setup, mock_heartbeat,
+                                         mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:api')
+
+        api.main()
+
+        mock_read_config.assert_called_with('designate', mock.ANY)
+        mock_log_setup.assert_called_with(mock.ANY, 'designate')
+        mock_service.assert_called_with()
+        mock_heartbeat.assert_called()
+        mock_serve.assert_called_with(mock.ANY, workers=1)
+        mock_wait.assert_called_with()
+
+    @mock.patch('designate.api.service.Service')
+    def test_api_heartbeat_stops_on_exception(
+            self, mock_service, mock_read_config, mock_log_setup,
+            mock_heartbeat, mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:api')
+        mock_wait.side_effect = KeyboardInterrupt()
+        mock_emitter = mock.Mock()
+        mock_heartbeat.return_value = mock_emitter
+
+        # call api.main and make sure it gets an exception
+        self.assertRaises(KeyboardInterrupt, api.main)
+
+        mock_emitter.stop.assert_called_once()
+
+    @mock.patch('designate.worker.service.Service')
+    def test_worker_init_host_called(self, mock_service, mock_read_config,
+                                     mock_log_setup, mock_heartbeat,
+                                     mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:worker')
+        mock_server = mock.Mock()
+        mock_service.return_value = mock_server
+
+        worker.main()
+
+        mock_server.init_host.assert_called_once()
+        mock_heartbeat.assert_called_with(mock_server.service_name)
+
+    @mock.patch('designate.producer.service.Service')
+    def test_producer_init_host_called(self, mock_service, mock_read_config,
+                                       mock_log_setup, mock_heartbeat,
+                                       mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:producer')
+        mock_server = mock.Mock()
+        mock_service.return_value = mock_server
+
+        producer.main()
+
+        mock_server.init_host.assert_called_once()
+        mock_heartbeat.assert_called_with(mock_server.service_name)
+
+    @mock.patch('designate.central.service.Service')
+    def test_central_rpc_already_initialized(
+            self, mock_service, mock_read_config, mock_log_setup,
+            mock_heartbeat, mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:central')
+
+        central.main()
+
+        mock_serve.assert_called_with(mock.ANY, workers=1)
+
+    @mock.patch('designate.mdns.service.Service')
+    def test_mdns_rpc_already_initialized(
+            self, mock_service, mock_read_config, mock_log_setup,
+            mock_heartbeat, mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:mdns')
+
+        mdns.main()
+
+        mock_serve.assert_called_with(mock.ANY, workers=1)
+
+    @mock.patch('designate.producer.service.Service')
+    def test_producer_rpc_already_initialized(
+            self, mock_service, mock_read_config, mock_log_setup,
+            mock_heartbeat, mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:producer')
+
+        producer.main()
+
+        mock_serve.assert_called_with(mock.ANY, workers=1)
+
+    @mock.patch('designate.sink.service.Service')
+    def test_sink_rpc_already_initialized(
+            self, mock_service, mock_read_config, mock_log_setup,
+            mock_heartbeat, mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:sink')
+
+        sink.main()
+
+        mock_serve.assert_called_with(mock.ANY, workers=1)
+
+    @mock.patch('designate.worker.service.Service')
+    def test_worker_rpc_already_initialized(
+            self, mock_service, mock_read_config, mock_log_setup,
+            mock_heartbeat, mock_serve, mock_wait):
+        CONF.set_override('workers', 1, 'service:worker')
+
+        worker.main()
+
+        mock_serve.assert_called_with(mock.ANY, workers=1)

designate/tests/unit/cmd/test_manage.py

@@ -15,7 +15,7 @@ from unittest import mock
 from oslo_config import fixture as cfg_fixture
 import oslotest.base
 
-from designate.cmd.threading import manage
+from designate.cmd import manage
 import designate.conf
 from designate.manage import base
 

designate/tests/unit/cmd/test_status.py

@@ -0,0 +1,30 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+from unittest import mock
+
+import oslotest.base
+
+from designate.cmd import status
+
+
+class StatusTestCase(oslotest.base.BaseTestCase):
+    @mock.patch('designate.cmd.status.upgradecheck.main')
+    @mock.patch('designate.cmd.status.utils.find_config')
+    def test_main(self, mock_find_config, mock_upgradecheck_main):
+        mock_find_config.return_value = ['/etc/designate/designate.conf']
+        mock_upgradecheck_main.return_value = 0
+
+        result = status.main()
+
+        self.assertEqual(0, result)
+        mock_find_config.assert_called_once_with('designate.conf')
+        mock_upgradecheck_main.assert_called_once()

designate/tests/unit/test_heartbeat_emitter.py

@@ -11,7 +11,6 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-import time
 from unittest import mock
 
 from oslo_config import fixture as cfg_fixture
@@ -69,11 +68,7 @@ class HeartbeatEmitterTest(oslotest.base.BaseTestCase):
     def test_emit(self):
         noop_emitter = heartbeat_emitter.get_heartbeat_emitter('svc')
 
-        noop_emitter.start()
-
-        time.sleep(0.125)
-
-        noop_emitter.stop()
+        noop_emitter._emit_heartbeat()
 
         self.assertIn(
             "<ServiceStatus service_name:'svc' hostname:'203.0.113.1' "

designate/tests/unit/test_service.py

@@ -176,6 +176,21 @@ class TestRpcService(oslotest.base.BaseTestCase):
     def test_rpc_service_wait(self):
         self.assertIsNone(self.service.wait())
 
+    @mock.patch.object(rpc, 'get_server')
+    @mock.patch.object(rpc, 'get_notifier')
+    def test_rpc_service_init_host_called_twice(
+            self, mock_rpc_get_notifier, mock_rpc_get_server):
+        # First call should initialize RPC
+        self.service.init_host()
+        mock_rpc_get_server.assert_called_once()
+        mock_rpc_get_notifier.assert_called_once()
+
+        # Second call should return early without re-initializing
+        self.service.init_host()
+        # Assert still only called once (not called again)
+        mock_rpc_get_server.assert_called_once()
+        mock_rpc_get_notifier.assert_called_once()
+
 
 @mock.patch.object(policy, 'init', mock.Mock())
 @mock.patch.object(rpc, 'init', mock.Mock())

designate/tests/unit/wsgi/test_api.py

@@ -1,7 +1,3 @@
-# Copyright 2012 Managed I.T.
-#
-# Author: Kiall Mac Innes <kiall@managedit.ie>
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
@@ -13,17 +9,19 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
-import os
+from unittest import mock
 
-# Eventlet's GreenDNS Patching will prevent the resolution of names in
-# the /etc/hosts file, causing problems for installs.
-os.environ['EVENTLET_NO_GREENDNS'] = 'yes'
+import oslotest.base
 
-import eventlet  # noqa
 
-eventlet.monkey_patch(os=False)
+class WSGIApiTestCase(oslotest.base.BaseTestCase):
+    @mock.patch('designate.api.wsgi.init_application')
+    def test_wsgi_api_application(self, mock_init_app):
+        mock_init_app.return_value = mock.Mock()
 
-import oslo_messaging as messaging  # noqa
+        # Import the module to test the module-level code
+        import designate.wsgi.api
 
-# Set some Oslo RPC defaults
-messaging.set_transport_defaults('designate')
+        # Verify the application was initialized
+        self.assertIsNotNone(designate.wsgi.api.application)
+        mock_init_app.assert_called_once()

designate/utils.py

@@ -304,7 +304,6 @@ def bind_tcp(host, port, tcp_backlog, tcp_keepidle=None):
     except Exception:
         LOG.info('SO_REUSEPORT not available, ignoring.')
 
-    # This option isn't available in the OS X version of eventlet
     if tcp_keepidle and hasattr(socket, 'TCP_KEEPIDLE'):
         sock_tcp.setsockopt(socket.IPPROTO_TCP,
                             socket.TCP_KEEPIDLE,

releasenotes/notes/eventlet-removal-complete-af40fe2f8dce994a.yaml

@@ -0,0 +1,11 @@
+---
+upgrade:
+  - |
+    Complete removal of eventlet from Designate. All services now use
+    native Python threading via oslo.service's threading backend.
+  - |
+    Updated oslo.service dependency to >=4.2.0 with the [threading] extra,
+    which is required for the threading backend support.
+  - |
+    Default thread counts have been adjusted to reduce potential memory
+    issues with Python native threads compared to eventlet greenthreads.

requirements.txt

@@ -2,9 +2,7 @@
 # date but we do not test them so no guarantee of having them all correct. If
 # you find any incorrect lower bounds, let us know or propose a fix.
 alembic>=1.8.0 # MIT
-eventlet>=0.36.0 # MIT
 Flask!=0.11,>=0.10 # BSD
-greenlet>=0.4.15 # MIT
 Jinja2>=2.10 # BSD License (3 clause)
 jsonschema>=3.2.0 # MIT
 keystoneauth1>=3.4.0 # Apache-2.0
@@ -18,7 +16,7 @@ oslo.log>=4.3.0 # Apache-2.0
 oslo.reports>=1.18.0 # Apache-2.0
 oslo.rootwrap>=5.15.0 # Apache-2.0
 oslo.serialization>=2.25.0 # Apache-2.0
-oslo.service>=1.31.0 # Apache-2.0
+oslo.service[threading]>=4.2.0 # Apache-2.0
 oslo.upgradecheck>=1.3.0
 oslo.utils>=4.7.0 # Apache-2.0
 oslo.versionedobjects>=1.31.2 # Apache-2.0

setup.cfg

@@ -54,14 +54,14 @@ oslo.policy.enforcer =
 
 console_scripts =
     designate-rootwrap = oslo_rootwrap.cmd:main
-    designate-api = designate.cmd.threading.api:main
-    designate-central = designate.cmd.threading.central:main
-    designate-manage = designate.cmd.threading.manage:main
-    designate-mdns = designate.cmd.threading.mdns:main
-    designate-sink = designate.cmd.threading.sink:main
-    designate-worker = designate.cmd.threading.worker:main
-    designate-producer = designate.cmd.threading.producer:main
-    designate-status = designate.cmd.threading.status:main
+    designate-api = designate.cmd.api:main
+    designate-central = designate.cmd.central:main
+    designate-manage = designate.cmd.manage:main
+    designate-mdns = designate.cmd.mdns:main
+    designate-sink = designate.cmd.sink:main
+    designate-worker = designate.cmd.worker:main
+    designate-producer = designate.cmd.producer:main
+    designate-status = designate.cmd.status:main
 
 designate.api.admin.extensions =
     reports = designate.api.admin.controllers.extensions.reports:ReportsController