openstack/glance
Code Issues Proposed changes

Make wsgi_app support graceful shutdown

Browse Source 
This makes us wait for all running threads in the tasks_pool threadpool
before exiting. Tested with uwsgi via devstack, with a long-running
image conversion task. Indeed, uwsgi triggers a graceful shutdown, we
wait for the remaining thread, and then exit when the conversion has
completed.

Closes-Bug: #1888713
Change-Id: I0cec7771ab8ce471607825eb4721fee1c6bdd1e6
This commit is contained in:
Dan Smith 2020-07-22 10:03:58 -07:00
parent 783fa72f48
commit f4b78606ba
2 changed files with 39 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
glance/common/wsgi_app.py
View File

@ -10,6 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import atexit
import os
import glance_store
@ -28,6 +29,7 @@ CONF = cfg.CONF
CONF.import_group("profiler", "glance.common.wsgi")
CONF.import_opt("enabled_backends", "glance.common.wsgi")
logging.register_options(CONF)
LOG = logging.getLogger(__name__)
CONFIG_FILES = ['glance-api-paste.ini',
'glance-image-import.conf',
@ -67,6 +69,16 @@ def _setup_os_profiler():
host=CONF.bind_host)
def drain_threadpools():
# NOTE(danms): If there are any other named pools that we need to
# drain before exit, they should be in this list.
pools_to_drain = ['tasks_pool']
for pool_name in pools_to_drain:
pool_model = common.get_thread_pool(pool_name)
LOG.info('Waiting for remaining threads in pool %r', pool_name)
pool_model.pool.shutdown()
def init_app():
config.set_config_defaults()
config_files = _get_config_files()
@ -76,6 +88,7 @@ def init_app():
# NOTE(danms): We are running inside uwsgi or mod_wsgi, so no eventlet;
# use native threading instead.
glance.async_.set_threadpool_model('native')
atexit.register(drain_threadpools)
# NOTE(danms): Change the default threadpool size since we
# are dealing with native threads and not greenthreads.

26
glance/tests/unit/common/test_wsgi_app.py
View File

@ -17,6 +17,7 @@
from unittest import mock
from glance.api import common
import glance.async_
from glance.common import wsgi_app
from glance.tests import utils as test_utils
@ -37,3 +38,28 @@ class TestWsgiAppInit(test_utils.BaseTestCase):
# Make sure we set the default pool size
self.assertEqual(123, common.DEFAULT_POOL_SIZE)
mock_load.assert_called_once_with('glance-api')
@mock.patch('atexit.register')
@mock.patch('glance.common.config.load_paste_app')
@mock.patch('glance.async_.set_threadpool_model')
@mock.patch('glance.common.wsgi_app._get_config_files')
def test_wsgi_init_registers_exit_handler(self, mock_config_files,
mock_set_model,
mock_load, mock_exit):
mock_config_files.return_value = []
wsgi_app.init_app()
mock_exit.assert_called_once_with(wsgi_app.drain_threadpools)
@mock.patch('glance.async_._THREADPOOL_MODEL', new=None)
def test_drain_threadpools(self):
# Initialize the thread pool model and tasks_pool, like API
# under WSGI would, and so we have a pointer to that exact
# pool object in the cache
glance.async_.set_threadpool_model('native')
model = common.get_thread_pool('tasks_pool')
with mock.patch.object(model.pool, 'shutdown') as mock_shutdown:
wsgi_app.drain_threadpools()
# Make sure that shutdown() was called on the tasks_pool
# ThreadPoolExecutor
mock_shutdown.assert_called_once_with()