From f4b78606bafd0c118de67f6e1ccdad8e0c5f555d Mon Sep 17 00:00:00 2001 From: Dan Smith Date: Wed, 22 Jul 2020 10:03:58 -0700 Subject: [PATCH] Make wsgi_app support graceful shutdown This makes us wait for all running threads in the tasks_pool threadpool before exiting. Tested with uwsgi via devstack, with a long-running image conversion task. Indeed, uwsgi triggers a graceful shutdown, we wait for the remaining thread, and then exit when the conversion has completed. Closes-Bug: #1888713 Change-Id: I0cec7771ab8ce471607825eb4721fee1c6bdd1e6 --- glance/common/wsgi_app.py | 13 ++++++++++++ glance/tests/unit/common/test_wsgi_app.py | 26 +++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/glance/common/wsgi_app.py b/glance/common/wsgi_app.py index 2f023912c1..d91952e501 100644 --- a/glance/common/wsgi_app.py +++ b/glance/common/wsgi_app.py @@ -10,6 +10,7 @@ # License for the specific language governing permissions and limitations # under the License. +import atexit import os import glance_store @@ -28,6 +29,7 @@ CONF = cfg.CONF CONF.import_group("profiler", "glance.common.wsgi") CONF.import_opt("enabled_backends", "glance.common.wsgi") logging.register_options(CONF) +LOG = logging.getLogger(__name__) CONFIG_FILES = ['glance-api-paste.ini', 'glance-image-import.conf', @@ -67,6 +69,16 @@ def _setup_os_profiler(): host=CONF.bind_host) +def drain_threadpools(): + # NOTE(danms): If there are any other named pools that we need to + # drain before exit, they should be in this list. + pools_to_drain = ['tasks_pool'] + for pool_name in pools_to_drain: + pool_model = common.get_thread_pool(pool_name) + LOG.info('Waiting for remaining threads in pool %r', pool_name) + pool_model.pool.shutdown() + + def init_app(): config.set_config_defaults() config_files = _get_config_files() @@ -76,6 +88,7 @@ def init_app(): # NOTE(danms): We are running inside uwsgi or mod_wsgi, so no eventlet; # use native threading instead. glance.async_.set_threadpool_model('native') + atexit.register(drain_threadpools) # NOTE(danms): Change the default threadpool size since we # are dealing with native threads and not greenthreads. diff --git a/glance/tests/unit/common/test_wsgi_app.py b/glance/tests/unit/common/test_wsgi_app.py index b58a8ac354..f9e0a7687b 100644 --- a/glance/tests/unit/common/test_wsgi_app.py +++ b/glance/tests/unit/common/test_wsgi_app.py @@ -17,6 +17,7 @@ from unittest import mock from glance.api import common +import glance.async_ from glance.common import wsgi_app from glance.tests import utils as test_utils @@ -37,3 +38,28 @@ class TestWsgiAppInit(test_utils.BaseTestCase): # Make sure we set the default pool size self.assertEqual(123, common.DEFAULT_POOL_SIZE) mock_load.assert_called_once_with('glance-api') + + @mock.patch('atexit.register') + @mock.patch('glance.common.config.load_paste_app') + @mock.patch('glance.async_.set_threadpool_model') + @mock.patch('glance.common.wsgi_app._get_config_files') + def test_wsgi_init_registers_exit_handler(self, mock_config_files, + mock_set_model, + mock_load, mock_exit): + mock_config_files.return_value = [] + wsgi_app.init_app() + mock_exit.assert_called_once_with(wsgi_app.drain_threadpools) + + @mock.patch('glance.async_._THREADPOOL_MODEL', new=None) + def test_drain_threadpools(self): + # Initialize the thread pool model and tasks_pool, like API + # under WSGI would, and so we have a pointer to that exact + # pool object in the cache + glance.async_.set_threadpool_model('native') + model = common.get_thread_pool('tasks_pool') + + with mock.patch.object(model.pool, 'shutdown') as mock_shutdown: + wsgi_app.drain_threadpools() + # Make sure that shutdown() was called on the tasks_pool + # ThreadPoolExecutor + mock_shutdown.assert_called_once_with()