Unify the thread pool executors into a central factory

Currently thread pool executors creation, destroy, get are
in utils and their singleton-ness is achieved via globals
variables which makes them less readable as utils.py is
becoming large. Also, there is no way to know what all
executors are created for this service/process which makes
hard perform single operation on all of them. For example,
shutdown all executors during graceful shutdown.

To solve the above issues, we are unifying the executors
into a single factory who will be responsible for their
creation, tracking, shutdown, and get. This will make sure
those are singleton per process and can be shutdown all
together.

This change is big in size as it moves the things from utils
to new file. A few key things changed here:

- Replace the per-manager executor attributes (e.g.
ComputeManager._live_migration_executor, _long_task_executor,
_sync_power_executor, and the scatter-gather/cache-images/default
executors scattered across nova.utils) with a single
ThreadPoolFactory that creates and tracks every executor as
a process-wide singleton, keyed by ExecutorType. In production,
no change as manager is a single instance per process but our
tests create multiple managers within same process.

- Gives get_executor()/spawn()/spawn_on() one common entry point
for every caller (manager etc) and gives shutdown_all_executors()
a single place to shut all of them. This makes usage easy as
caller does not need to worry about executors creation/get when
they want to spawn the things.

- spawn_on takes executor type as arg and then create/get the
executors by its own. Callers just need to pass what type of
executors they want to spawn things on.

- pool size which are mostly from config options are also
maintained in central place.

- A single method to shutdown and reset all the executors created
in specific process.

- ComputeManager's _cleanup_live_migrations_in_pool() no longer
shutdown its own executor directly, since the live migration
executor is now shared across every compute manager instance in
the process rather than owned by one. And RPC server stop will
take care that no new live migration is heppning during this
cleanup.

Change-Id: Ifcb4b4ad1113d742e26d91cbb35e5b92a14878e3
Signed-off-by: Ghanshyam Maan <gmaan.os14@gmail.com>
This commit is contained in:
Ghanshyam Maan committed 2026-08-18 14:33:42 -07:00
1 parent 2a745c7e15
commit 0842edaa99
27 files changed
+2191 -1699

No files matched your search

+2 -3
View File
@@ -29,7 +29,7 @@ import nova.conf
from nova import config
from nova import objects
from nova import service
from nova import utils
from nova import thread_pool_factory
from nova import version
CONF = nova.conf.CONF
@@ -48,7 +48,6 @@ def main():
topic=rpcapi.RPC_TOPIC)
workers = CONF.conductor.workers or processutils.get_worker_count()
utils.destroy_default_executor()
utils.destroy_scatter_gather_executor()
thread_pool_factory.reset_all_executors()
service.serve(server, workers=workers)
service.wait()
+3 -4
View File
@@ -31,7 +31,7 @@ from nova import config
from nova import objects
from nova.scheduler import rpcapi
from nova import service
from nova import utils
from nova import thread_pool_factory
from nova import version
CONF = nova.conf.CONF
@@ -58,10 +58,9 @@ def main():
# parent to the child process, we destroy the executor in the parent
# process before the forking, so that the workers initialize new
# executor(s) and therefore avoid working with the wrong internal executor
# state (i.e. number of workers idle in the pool). A long therm solution
# state (i.e. number of workers idle in the pool). A long term solution
# would be to use os.spawn instead of os.fork for the workers.
utils.destroy_scatter_gather_executor()
utils.destroy_default_executor()
thread_pool_factory.reset_all_executors()
service.serve(server, workers=workers)
service.wait()
+29 -114
View File
@@ -88,6 +88,7 @@ from nova.scheduler.client import query
from nova.scheduler.client import report
from nova.scheduler import utils as scheduler_utils
from nova.share import manila
from nova import thread_pool_factory
from nova import utils
from nova.virt import block_device as driver_block_device
from nova.virt import configdrive
@@ -666,57 +667,26 @@ class ComputeManager(manager.Manager):
self.compute_task_api = conductor.ComputeTaskAPI()
self.query_client = query.SchedulerQueryClient()
self.instance_events = InstanceEvents()
self._sync_power_executor = nova.utils.create_executor(
max_workers=CONF.sync_power_state_pool_size)
self._syncs_in_progress: set[str] = set()
self._syncs_in_progress_lock = threading.Lock()
self.send_instance_updates = (
CONF.filter_scheduler.track_instance_changes)
max_builds = self._get_max_concurrent_builds()
max_snapshots = self._get_max_concurrent_snapshots()
if utils.concurrency_mode_threading():
max_tasks = max(max_builds, max_snapshots)
if max_builds != max_snapshots:
LOG.warning(
"In native threading mode the number of concurrent "
"builds, and snapshots should be limited to the "
"same number. The current configuration has differing "
"limits: max_concurrent_builds: %d, "
"max_concurrent_snapshots: %d. "
"Nova will use a single, overall limit of %d for these "
"tasks.",
max_builds, max_snapshots, max_tasks)
self._long_task_executor = utils.get_long_task_executor(max_tasks)
# In threading mode we want to use the size of the executor to
# act as the limit of concurrent execution. So neuter the
# semaphores here.
# TODO(gibi): remove the semaphores once eventlet mode is removed
self._build_semaphore = compute_utils.UnlimitedSemaphore()
self._snapshot_semaphore = compute_utils.UnlimitedSemaphore()
else:
# In eventlet mode we use the individual semaphores to limit
# the concurrent tasks, so just create a big Executor to
# potentially host all of them
self._long_task_executor = utils.get_long_task_executor(
max_builds + max_snapshots)
self._build_semaphore = threading.Semaphore(max_builds)
self._snapshot_semaphore = threading.Semaphore(max_snapshots)
# While live migration is a long-running task we cannot put it into
# the same long_task_executor as build and snapshot as we need:
# 1. a very small limit of concurrent live migrations compared to
# builds and snapshots
# 2. a way to cancel live migrations easily that are waiting due to the
# limit
self._live_migration_executor = nova.utils.create_executor(
max_workers=self._get_max_concurrent_live_migrations())
self._build_semaphore = threading.Semaphore(
thread_pool_factory.get_max_concurrent_builds())
self._snapshot_semaphore = threading.Semaphore(
thread_pool_factory.get_max_concurrent_snapshots())
# This is a dict, keyed by instance uuid, to a two-item tuple of
# migration object and Future for the queued live migration.
@@ -751,68 +721,6 @@ class ComputeManager(manager.Manager):
self.rt = resource_tracker.ResourceTracker(
self.host, self.driver, reportclient=self.reportclient)
def _get_max_concurrent_builds(self):
if CONF.max_concurrent_builds > 0:
return CONF.max_concurrent_builds
# setting CONF.max_concurrent_builds to 0 (unlimited)
# is deprecated but still supported, so we need to use a sane
# default values for each threading mode
LOG.warning("Nova compute deprecated the support of unlimited "
"parallel instance builds so "
"[DEFAULT]max_concurrent_builds configured "
"with value 0 is deprecated and will not be supported "
"in future releases. Please set an explicit positive "
"value to this config option instead.")
if utils.concurrency_mode_threading():
# Fall back to the default of the config
return 10
else:
# In eventlet mode we need to keep backward compatibility, and
# we use 1000 to emulate unlimited
return 1000
def _get_max_concurrent_snapshots(self):
if CONF.max_concurrent_snapshots > 0:
return CONF.max_concurrent_snapshots
# setting CONF.max_concurrent_snapshots to 0 (unlimited)
# is deprecated but still supported, so we need to use a sane
# default values for each threading mode
LOG.warning("Nova compute deprecated the support of unlimited "
"parallel instance snapshots so "
"[DEFAULT]max_concurrent_snapshots configured "
"with value 0 is deprecated and will not be supported "
"in future releases. Please set an explicit positive "
"value to this config option instead.")
if utils.concurrency_mode_threading():
# Fall back to the default of the config
return 5
else:
# In eventlet mode we need to keep backward compatibility, and
# we use 1000 to emulate unlimited
return 1000
def _get_max_concurrent_live_migrations(self):
if CONF.max_concurrent_live_migrations > 0:
return CONF.max_concurrent_live_migrations
# setting CONF.max_concurrent_live_migrations to 0 (unlimited)
# is deprecated but still supported, so we need to use a sane
# default values for each threading mode
LOG.warning("Nova compute deprecated the support of unlimited "
"parallel live migration so "
"[DEFAULT]max_concurrent_live_migrations configured "
"with value 0 is deprecated and will not be supported "
"in future releases. Please set an explicit positive"
"value to this config option instead.")
if utils.concurrency_mode_threading():
return 5
else:
# In eventlet mode we need to keep backward compatibility and
# 1000 greenthreads to emulate unlimited
return 1000
@contextlib.contextmanager
def syncs_in_progress(self) -> Iterator[set[str]]:
with self._syncs_in_progress_lock:
@@ -1852,9 +1760,11 @@ class ComputeManager(manager.Manager):
def _cleanup_live_migrations_in_pool(self):
# Shutdown the pool so we don't get new requests.
self._live_migration_executor.shutdown(wait=False)
# For any queued migrations, cancel the migration and update
# its status.
live_migration_executor = thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.LIVE_MIGRATION)
live_migration_executor.shutdown(wait=False)
thread_pool_factory.FACTORY._all_executors.pop(
thread_pool_factory.ExecutorType.LIVE_MIGRATION, None)
for migration, future in self._waiting_live_migrations.values():
# If we got here before the Future was submitted then we need
# to move on since there isn't anything we can do.
@@ -2421,8 +2331,8 @@ class ComputeManager(manager.Manager):
def _build_succeeded(self, node):
self.rt.build_succeeded(node)
def _start_recorded_task(self, executor, task_name, instance, context,
func, *args, **kwargs):
def _start_recorded_task(self, executor_type, task_name, instance,
context, func, *args, **kwargs):
"""Spawn func on executor while tracking it as an in-progress task.
Wraps the common pattern of recording a task as started, spawning
@@ -2430,19 +2340,21 @@ class ComputeManager(manager.Manager):
itself raises, and arranging for it to be recorded as completed or
failed (based on the Future's outcome) when it finishes.
:param executor: the executor to spawn func on
:param executor_type: the thread_pool_factory.ExecutorType to spawn
func on
:param task_name: task name, most of time it is RPC method name
:param instance: the nova.objects.Instance the task is for
:param context: the request context, used for the request id
:param func: the callable to spawn
:param args: positional arguments passed to func
:param kwargs: keyword arguments passed to func
:returns: the Future returned by utils.spawn_on()
:returns: the Future returned by thread_pool_factory.spawn_on()
"""
key = self._record_task_start(
task_name, instance.uuid, context.request_id)
try:
future = utils.spawn_on(executor, func, *args, **kwargs)
future = thread_pool_factory.spawn_on(
executor_type, func, *args, **kwargs)
except Exception:
self._record_task_end(key, failed=True)
raise
@@ -2517,7 +2429,8 @@ class ComputeManager(manager.Manager):
# NOTE(gmaan): Track this task so that graceful_shutdown() can wait for
# the build to complete.
self._start_recorded_task(
self._long_task_executor, 'build_instance', instance, context,
thread_pool_factory.ExecutorType.LONG_TASK, 'build_instance',
instance, context,
_locked_do_build_and_run_instance,
context, instance, image, request_spec,
filter_properties, admin_password, injected_files,
@@ -4700,9 +4613,9 @@ class ComputeManager(manager.Manager):
# avoid blocking RPC traffic.
# Track this as an active task for graceful shutdown.
return self._start_recorded_task(
self._long_task_executor, 'snapshot_instance', instance, context,
do_snapshot_instance, context, image_id, instance,
task_states.IMAGE_SNAPSHOT)
thread_pool_factory.ExecutorType.LONG_TASK, 'snapshot_instance',
instance, context, do_snapshot_instance, context, image_id,
instance, task_states.IMAGE_SNAPSHOT)
@wrap_exception()
@reverts_task_state
@@ -9914,9 +9827,10 @@ class ComputeManager(manager.Manager):
# decremented via callback when the future finishes.
try:
future = self._start_recorded_task(
self._live_migration_executor, 'live_migration', instance,
context, self._do_live_migration, context, dest, instance,
block_migration, migration, migrate_data)
thread_pool_factory.ExecutorType.LIVE_MIGRATION,
'live_migration', instance, context, self._do_live_migration,
context, dest, instance, block_migration, migration,
migrate_data)
except RuntimeError:
# GreenThreadPoolExecutor.submit will raise RuntimeError if the
# pool is shutdown, which happens in
@@ -11261,8 +11175,9 @@ class ComputeManager(manager.Manager):
else:
LOG.debug('Triggering sync for uuid %s', uuid)
syncs.add(uuid)
nova.utils.spawn_on(
self._sync_power_executor, _sync, db_instance)
thread_pool_factory.spawn_on(
thread_pool_factory.ExecutorType.SYNC_POWER,
_sync, db_instance)
def _query_driver_power_state_and_sync(self, context, db_instance):
if db_instance.task_state is not None:
+4 -4
View File
@@ -62,6 +62,7 @@ from nova.scheduler.client import query
from nova.scheduler.client import report
from nova.scheduler import utils as scheduler_utils
from nova import servicegroup
from nova import thread_pool_factory
from nova import utils
from nova.volume import cinder
@@ -2082,7 +2083,6 @@ class ComputeTaskManager:
fields.NotificationPhase.START)
clock = timeutils.StopWatch()
cache_image_executor = utils.get_cache_images_executor()
futures = []
hosts_by_cell = {}
@@ -2154,9 +2154,9 @@ class ComputeTaskManager:
{'host': host})
skipped_host(target_ctxt, host, image_ids)
continue
future = utils.spawn_on(cache_image_executor,
wrap_cache_images,
target_ctxt, host, image_ids)
future = thread_pool_factory.spawn_on(
thread_pool_factory.ExecutorType.CACHE_IMAGES,
wrap_cache_images, target_ctxt, host, image_ids)
futures.append(future)
# Wait until all those things finish
concurrent.futures.wait(futures)
+5 -5
View File
@@ -34,6 +34,7 @@ from nova.i18n import _
from nova import objects
from nova import policy
from nova import service_auth
from nova import thread_pool_factory
from nova import utils
LOG = logging.getLogger(__name__)
@@ -463,13 +464,12 @@ def scatter_gather_cells(context, cell_mappings, timeout, fn, *args, **kwargs):
return result
executor = utils.get_scatter_gather_executor()
for cell_mapping in cell_mappings:
with target_cell(context, cell_mapping) as cctxt:
future = utils.spawn_on(
executor,
gather_result, cell_mapping.uuid, fn, cctxt, *args, **kwargs)
future = thread_pool_factory.spawn_on(
thread_pool_factory.ExecutorType.SCATTER_GATHER,
gather_result, cell_mapping.uuid, fn, cctxt, *args,
**kwargs)
tasks[cell_mapping.uuid] = future
futurist.waiters.wait_for_all(tasks.values(), timeout)
+2 -2
View File
@@ -585,9 +585,9 @@ def check_greenthread_spawns(logical_line, filename):
N340
"""
msg = ("N340: Use nova.utils.spawn() rather than "
msg = ("N340: Use nova.thread_pool_factory.spawn() rather than "
"greenthread.%(spawn)s() and eventlet.%(spawn)s()")
if "nova/utils.py" in filename or "nova/tests/" in filename:
if "nova/thread_pool_factory.py" in filename or "nova/tests/" in filename:
return
match = re.match(spawn_re, logical_line)
+2 -2
View File
@@ -20,7 +20,7 @@ from oslo_serialization import jsonutils
from nova import exception
from nova.i18n import _
from nova import utils
from nova import thread_pool_factory
# Constants for the 'vif_type' field in VIF class
@@ -583,7 +583,7 @@ class NetworkInfoAsyncWrapper(NetworkInfo):
def __init__(self, async_method, *args, **kwargs):
super(NetworkInfoAsyncWrapper, self).__init__()
self._future = utils.spawn(async_method, *args, **kwargs)
self._future = thread_pool_factory.spawn(async_method, *args, **kwargs)
methods = ['json', 'fixed_ips', 'floating_ips']
for method in methods:
fn = getattr(self, method)
+2 -1
View File
@@ -31,6 +31,7 @@ from nova import objects
from nova.pci import stats as pci_stats
from nova.scheduler import filters
from nova.scheduler import weights
from nova import thread_pool_factory
from nova import utils
from nova.virt import hardware
@@ -468,7 +469,7 @@ class HostManager(object):
LOG.debug("END:_async_init_instance_info")
# Run this async so that we don't block the scheduler start-up
utils.spawn(_async_init_instance_info, computes_by_cell)
thread_pool_factory.spawn(_async_init_instance_info, computes_by_cell)
def _choose_host_filters(self, filter_cls_names):
"""Since the caller may specify which filters to use we need
+2 -1
View File
@@ -155,7 +155,8 @@ class FakeVersionedNotifier(FakeNotifier):
current = self.eventlet.getcurrent()
# NOTE(gibi) not all eventlet spawn is under our control, so there can
# be senders without test_case_id set, find the first ancestor that
# was spawned from nova.utils.spawn[_n] and therefore has the id set.
# was spawned from nova.thread_pool_factory.spawn[_n] and therefore has
# the id set.
while not getattr(current, 'test_case_id', None):
current = current.parent
return current.test_case_id
+52 -110
View File
@@ -69,6 +69,7 @@ from nova import rpc
from nova.scheduler import weights
from nova import service
from nova.tests.functional.api import client
from nova import thread_pool_factory
from nova import utils
from nova.virt import node
@@ -1310,109 +1311,41 @@ class IsolatedExecutorFixture(fixtures.Fixture):
def _setUp(self):
# Just safety that the previous testcase cleaned up after itself
assert utils.SCATTER_GATHER_EXECUTOR is None
assert utils.DEFAULT_EXECUTOR is None
assert utils.CACHE_IMAGES_EXECUTOR is None
assert utils.LONG_TASK_EXECUTOR is None
assert not thread_pool_factory.FACTORY._all_executors
origi_get_scatter_gather = utils.get_scatter_gather_executor
origi_default_executor = utils._get_default_executor
origi_get_cache_images_executor = utils.get_cache_images_executor
origi_get_long_task_executor = utils.get_long_task_executor
origi_get_executor = thread_pool_factory.get_executor
self.executor = None
self.scatter_gather_executor = None
self.cache_images_executor = None
self.long_task_executor = None
def _get_executor(executor_type, **kwargs):
executor = origi_get_executor(executor_type, **kwargs)
executor.name = f"{self.test_case_id}.{executor_type.value}"
return executor
def _get_default_executor():
self.executor = origi_default_executor()
self.executor.name = f"{self.test_case_id}.default"
return self.executor
# NOTE(sean-k-mooney): greenpools use eventlet.spawn so we can't stub
# out all calls to those functions.
# Instead since nova only creates greenthreads directly via nova.utils
# we stub out the default green pool. This will not capture
# Greenthreads created via the standard lib threading module.
# Instead since nova only creates greenthreads directly via
# nova.thread_pool_factory we stub out the executor getter. This will
# not capture Greenthreads created via the standard lib threading
# module.
self.useFixture(fixtures.MonkeyPatch(
'nova.utils._get_default_executor', _get_default_executor))
self.addCleanup(lambda: self.do_cleanup_executor(self.executor))
'nova.thread_pool_factory.get_executor', _get_executor))
def _get_scatter_gather_executor():
self.scatter_gather_executor = origi_get_scatter_gather()
self.scatter_gather_executor.name = (
f"{self.test_case_id}.cell_worker")
return self.scatter_gather_executor
self.addCleanup(self._cleanup_executors)
self.useFixture(fixtures.MonkeyPatch(
'nova.utils.get_scatter_gather_executor',
_get_scatter_gather_executor))
self.addCleanup(
lambda: self.do_cleanup_executor(self.scatter_gather_executor))
def _get_cache_images_executor():
self.cache_images_executor = origi_get_cache_images_executor()
self.cache_images_executor.name = (
f"{self.test_case_id}.cache_images")
return self.cache_images_executor
self.useFixture(fixtures.MonkeyPatch(
'nova.utils.get_cache_images_executor',
_get_cache_images_executor))
self.addCleanup(
lambda: self.do_cleanup_executor(self.cache_images_executor))
def _get_long_task_executor(max_workers):
self.long_task_executor = origi_get_long_task_executor(max_workers)
self.long_task_executor.name = (
f"{self.test_case_id}.long_task")
return self.long_task_executor
self.useFixture(fixtures.MonkeyPatch(
'nova.utils.get_long_task_executor',
_get_long_task_executor))
self.addCleanup(
lambda: self.do_cleanup_executor(self.long_task_executor))
self.addCleanup(self.reset_globals)
def reset_globals(self):
utils.SCATTER_GATHER_EXECUTOR = None
utils.DEFAULT_EXECUTOR = None
utils.CACHE_IMAGES_EXECUTOR = None
utils.LONG_TASK_EXECUTOR = None
def do_cleanup_executor(self, executor):
# NOTE(gibi): we cannot rely on utils.concurrency_mode_threading
# as that might have been mocked during the test when the executor
# was created, but during cleanup the mock is already removed.
threading = isinstance(executor, futurist.ThreadPoolExecutor)
if executor and executor.alive:
executor.shutdown(wait=False)
if threading:
# NOTE(gibi): This is optimistic, but we need specific examples
# where self.executor.shutdown(wait=True) hangs to figure out
# what we can do here.
pass
else:
def _cleanup_executors(self):
executors = list(thread_pool_factory.FACTORY._all_executors.values())
# reset_all_executors call shutdown(wait=True) for all executors.
thread_pool_factory.reset_all_executors()
for executor in executors:
# NOTE(gibi): we cannot rely on utils.concurrency_mode_threading
# as that might have been mocked during the test when the executor
# was created, but during cleanup the mock is already removed.
if isinstance(executor, futurist.GreenThreadPoolExecutor):
eventlet = utils.get_eventlet()
# kill all greenthreads in the pool before raising to prevent
# them from interfering with other tests.
for gt in list(executor._pool.coroutines_running):
if isinstance(gt, eventlet.greenthread.GreenThread):
gt.kill()
executor.shutdown(wait=True)
if threading:
# NOTE(gibi):If shutdown(wait=True) returns then nothing is
# leaked, so nothing to do here.
pass
else:
self._raise_on_green_pool(executor._pool)
def _raise_on_green_pool(self, pool):
@@ -1451,16 +1384,18 @@ class SpawnIsSynchronousFixture(fixtures.Fixture):
executor = futurist.SynchronousExecutor()
self.addCleanup(executor.shutdown)
def spawn(*args, **kwargs):
return executor.submit(*args, **kwargs)
def spawn(func, *args, **kwargs):
return executor.submit(func, *args, **kwargs)
# Just ignore the first arg that is the original executor instance
# and use our test internal synchronous executor.
def spawn_on(_, *args, **kwargs):
return executor.submit(*args, **kwargs)
# Just ignore the executor_type argument and use our test internal
# synchronous executor instead.
def spawn_on(executor_type, func, *args, **kwargs):
return executor.submit(func, *args, **kwargs)
self.useFixture(fixtures.MonkeyPatch('nova.utils.spawn', spawn))
self.useFixture(fixtures.MonkeyPatch('nova.utils.spawn_on', spawn_on))
self.useFixture(fixtures.MonkeyPatch(
'nova.thread_pool_factory.spawn', spawn))
self.useFixture(fixtures.MonkeyPatch(
'nova.thread_pool_factory.spawn_on', spawn_on))
class BannedDBSchemaOperations(fixtures.Fixture):
@@ -1998,16 +1933,16 @@ class PropagateTestCaseIdToChildEventlets(fixtures.Fixture):
c = eventlet.getcurrent()
c.test_case_id = self.test_case_id
orig_spawn = utils.spawn
orig_spawn = thread_pool_factory.spawn
def wrapped_spawn(func, *args, **kwargs):
# This is still runs before the eventlet.spawn so read the id for
# propagation
caller = eventlet.getcurrent()
# If there is no id set on us that means we were spawned with other
# than nova.utils.spawn so the id propagation chain got
# broken. We fall back to self.test_case_id from the fixture which
# is good enough
# If there is no id set on us that means we were spawned with
# other than nova.thread_pool_factory.spawn so the id propagation
# chain got broken. We fall back to self.test_case_id from the
# fixture which is good enough
caller_test_case_id = getattr(
caller, 'test_case_id', None) or self.test_case_id
@@ -2023,10 +1958,11 @@ class PropagateTestCaseIdToChildEventlets(fixtures.Fixture):
# new wrapper around its target
return orig_spawn(test_case_id_wrapper, *args, **kwargs)
# let's replace nova.utils.spawn with the wrapped one that injects
# our initialization to the child eventlet
self.useFixture(
fixtures.MonkeyPatch('nova.utils.spawn', wrapped_spawn))
# let's replace nova.thread_pool_factory.spawn with the wrapped one
# that injects our initialization to the child eventlet
self.useFixture(fixtures.MonkeyPatch(
'nova.thread_pool_factory.spawn',
wrapped_spawn))
class ReaderWriterLock(lockutils.ReaderWriterLock):
@@ -2371,7 +2307,10 @@ class DelayingExecutorWrapperCleanupFixture(fixtures.Fixture):
def setUp(self):
super().setUp()
orig = utils.StaticallyDelayingCancellableTaskExecutorWrapper.__init__
orig = (
thread_pool_factory.
StaticallyDelayingCancellableTaskExecutorWrapper.
__init__)
def wrapped_init(executor_wrapper, delay, executor):
stack = "".join(traceback.format_stack())
@@ -2381,18 +2320,21 @@ class DelayingExecutorWrapperCleanupFixture(fixtures.Fixture):
self.useFixture(
fixtures.MonkeyPatch(
'nova.utils.StaticallyDelayingCancellableTaskExecutorWrapper.'
'nova.thread_pool_factory.'
'StaticallyDelayingCancellableTaskExecutorWrapper.'
'__init__', wrapped_init))
@staticmethod
def _check_wrapper_stopped(
wrapper: utils.StaticallyDelayingCancellableTaskExecutorWrapper,
wrapper: thread_pool_factory.
StaticallyDelayingCancellableTaskExecutorWrapper,
stack: str,
):
if wrapper.is_alive:
raise RuntimeError(
'The test case leaked an active '
'nova.utils.StaticallyDelayingCancellableTaskExecutorWrapper'
'nova.thread_pool_factory.'
'StaticallyDelayingCancellableTaskExecutorWrapper'
'instance. This can lead to unexpected failures in later test '
'case. Please ensure that shutdown(wait=true) is called on '
'the wrapper before the end of the test case e.g. by using '
@@ -37,7 +37,6 @@ import fixtures
from nova import test
from nova.tests import fixtures as nova_fixtures
from nova.tests.functional import integrated_helpers
from nova import utils
class GracefulShutdownTestBase(integrated_helpers.ProviderUsageBaseTestCase):
@@ -884,9 +883,6 @@ class TestConductorGracefulShutdown(test.TestCase):
manager = self.conductor_service.manager
self.assertFalse(manager._shutdown_in_progress.is_set())
utils.get_cache_images_executor()
self.addCleanup(utils.destroy_cache_images_executor)
start = time.monotonic()
self.conductor_service.stop()
elapsed = time.monotonic() - start
+15 -6
View File
@@ -15,7 +15,7 @@ from unittest import mock
from nova.cmd import scheduler
from nova import config
from nova import test
from nova import utils
from nova import thread_pool_factory
# required because otherwise oslo early parse_args dies
@@ -55,15 +55,24 @@ class TestScheduler(test.NoDBTestCase):
self, mock_wait, mock_serve, service_create
):
# simulate that the thread pool is initialized before the fork
executor = utils._get_default_executor()
sc_executor = utils.get_scatter_gather_executor()
executor = thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT)
sc_executor = thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.SCATTER_GATHER)
scheduler.main()
mock_serve.assert_called_once_with(
service_create.return_value, workers=mock.ANY)
mock_wait.assert_called_once_with()
# check that the executor was properly destroyed
# check that the executors were properly destroyed
self.assertFalse(executor.alive)
self.assertIsNone(utils.DEFAULT_EXECUTOR)
self.assertFalse(sc_executor.alive)
self.assertIsNone(utils.SCATTER_GATHER_EXECUTOR)
self.assertEqual({}, thread_pool_factory.FACTORY._all_executors)
# check that the factory is not left in a permanently
# "shutdown" state, since the process (and its forked workers)
# must still be able to create new executors afterwards
self.assertFalse(thread_pool_factory.FACTORY._shutdown)
new_executor = thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT)
self.addCleanup(new_executor.shutdown, wait=False)
+3 -2
View File
@@ -87,6 +87,7 @@ from nova.tests.unit.objects import test_flavor
from nova.tests.unit.objects import test_instance_numa
from nova.tests.unit.objects import test_migration
from nova.tests.unit import utils as test_utils
from nova import thread_pool_factory
from nova import utils
from nova.virt import block_device as driver_block_device
from nova.virt.block_device import DriverVolumeBlockDevice as driver_bdm_volume
@@ -1661,7 +1662,6 @@ class ComputeTestCase(BaseTestCase,
fake_resource_tracker.RTMockMixin):
def setUp(self):
super(ComputeTestCase, self).setUp()
self.compute._live_migration_executor = futurist.SynchronousExecutor()
# NOTE(gibi): the _sync_power_states periodic task in the
# ComputeManager spawning concurrent tasks and uses a lock to
# synchronize a shared data structure. As the spawn is made
@@ -8575,7 +8575,8 @@ class ComputeTestCase(BaseTestCase,
self.compute._sync_power_states(ctxt)
# wait for sync to finish
self.compute._sync_power_executor.shutdown(wait=True)
thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.SYNC_POWER).shutdown(wait=True)
mock_get.assert_has_calls([mock.call(mock.ANY), mock.call(mock.ANY),
mock.call(mock.ANY)])
+34 -47
View File
@@ -70,6 +70,7 @@ from nova.tests.unit import fake_network
from nova.tests.unit import fake_network_cache_model
from nova.tests.unit.objects import test_instance_fault
from nova.tests.unit.objects import test_instance_info_cache
from nova import thread_pool_factory
from nova import utils
from nova.virt.block_device import DriverVolumeBlockDevice as driver_bdm_volume
from nova.virt import driver as virt_driver
@@ -970,24 +971,20 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase,
self._test_max_concurrent_builds()
def test_max_concurrent_builds_semaphore_limited(self):
utils.destroy_long_task_executor()
self.flags(max_concurrent_builds=123)
compute = manager.ComputeManager()
if utils.concurrency_mode_threading():
self.assertIsInstance(
compute._build_semaphore, compute_utils.UnlimitedSemaphore)
self.assertEqual(123, compute._long_task_executor._max_workers)
else:
self.assertEqual(123, compute._build_semaphore._value)
def test_max_concurrent_builds_semaphore_unlimited(self):
utils.destroy_long_task_executor()
self.flags(max_concurrent_builds=0)
compute = manager.ComputeManager()
if utils.concurrency_mode_threading():
self.assertIsInstance(
compute._build_semaphore, compute_utils.UnlimitedSemaphore)
self.assertEqual(10, compute._long_task_executor._max_workers)
else:
self.assertEqual(1000, compute._build_semaphore._value)
@@ -1041,8 +1038,8 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase,
wraps=self.compute._record_task_end),
) as (mock_task_start, mock_task_end):
future = self.compute._start_recorded_task(
self.compute._long_task_executor, 'fake-task', instance,
self.context, func)
thread_pool_factory.ExecutorType.LONG_TASK, 'fake-task',
instance, self.context, func)
self.assertRaises(test.TestingException, future.result)
mock_task_end.assert_called_once_with(1, mock.ANY)
@@ -1059,12 +1056,14 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase,
mock.patch.object(
self.compute, '_record_task_end',
wraps=self.compute._record_task_end),
mock.patch('nova.utils.spawn_on', side_effect=RuntimeError),
mock.patch(
'nova.thread_pool_factory.spawn_on',
side_effect=RuntimeError),
) as (mock_task_start, mock_task_end, mock_spawn):
self.assertRaises(
RuntimeError, self.compute._start_recorded_task,
self.compute._long_task_executor, 'fake-task', instance,
self.context, func)
thread_pool_factory.ExecutorType.LONG_TASK, 'fake-task',
instance, self.context, func)
mock_task_start.assert_called_once_with(
'fake-task', instance.uuid, self.context.request_id)
@@ -1081,46 +1080,35 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase,
self._test_max_concurrent_snapshots()
def test_max_concurrent_snapshots_semaphore_limited(self):
utils.destroy_long_task_executor()
self.flags(max_concurrent_snapshots=123)
compute = manager.ComputeManager()
if utils.concurrency_mode_threading():
self.assertIsInstance(
compute._snapshot_semaphore, compute_utils.UnlimitedSemaphore)
self.assertEqual(123, compute._long_task_executor._max_workers)
else:
self.assertEqual(123, compute._snapshot_semaphore._value)
def test_max_concurrent_snapshots_semaphore_unlimited(self):
utils.destroy_long_task_executor()
self.flags(max_concurrent_snapshots=0)
compute = manager.ComputeManager()
if utils.concurrency_mode_threading():
self.assertIsInstance(
compute._snapshot_semaphore, compute_utils.UnlimitedSemaphore)
self.assertEqual(10, compute._long_task_executor._max_workers)
else:
self.assertEqual(1000, compute._snapshot_semaphore._value)
@mock.patch.object(manager.LOG, 'warning')
def test_max_c_builds_and_snapshots_different_limits(self, mock_log):
utils.destroy_long_task_executor()
def test_max_c_builds_and_snapshots_different_limits(self):
self.flags(max_concurrent_builds=124)
self.flags(max_concurrent_snapshots=123)
compute = manager.ComputeManager()
if utils.concurrency_mode_threading():
self.assertEqual(124, compute._long_task_executor._max_workers)
mock_log.assert_called_once_with(
'In native threading mode the number of concurrent builds, '
'and snapshots should be limited to the same number. '
'The current configuration has differing limits: '
'max_concurrent_builds: %d, max_concurrent_snapshots: %d. '
'Nova will use a single, overall limit of %d for these tasks.',
124, 123, 124)
self.assertIsInstance(
compute._snapshot_semaphore, compute_utils.UnlimitedSemaphore)
self.assertIsInstance(
compute._build_semaphore, compute_utils.UnlimitedSemaphore)
else:
self.assertEqual(123, compute._snapshot_semaphore._value)
self.assertEqual(124, compute._build_semaphore._value)
mock_log.assert_not_called()
def test_nil_out_inst_obj_host_and_node_sets_nil(self):
instance = fake_instance.fake_instance_obj(self.context,
@@ -1306,23 +1294,18 @@ class ComputeManagerUnitTestCase(test.NoDBTestCase,
self.compute._waiting_live_migrations[fake_instance_uuid] = (
fake_migration, fake_future)
with mock.patch.object(self.compute, '_live_migration_executor'
) as mock_migration_pool:
self.compute._cleanup_live_migrations_in_pool()
self.compute._cleanup_live_migrations_in_pool()
mock_migration_pool.shutdown.assert_called_once_with(wait=False)
self.assertEqual('cancelled', fake_migration.status)
fake_future.cancel.assert_called_once_with()
self.assertEqual({}, self.compute._waiting_live_migrations)
self.assertEqual('cancelled', fake_migration.status)
fake_future.cancel.assert_called_once_with()
self.assertEqual({}, self.compute._waiting_live_migrations)
# test again with Future is None
self.compute._waiting_live_migrations[fake_instance_uuid] = (
None, None)
self.compute._cleanup_live_migrations_in_pool()
# test again with Future is None
self.compute._waiting_live_migrations[fake_instance_uuid] = (
None, None)
self.compute._cleanup_live_migrations_in_pool()
mock_migration_pool.shutdown.assert_called_with(wait=False)
self.assertEqual(2, mock_migration_pool.shutdown.call_count)
self.assertEqual({}, self.compute._waiting_live_migrations)
self.assertEqual({}, self.compute._waiting_live_migrations)
def test_init_virt_events_disabled(self):
self.flags(handle_virt_lifecycle_events=False, group='workarounds')
@@ -11900,7 +11883,7 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase,
self.assertEqual('queued', migration.status)
migration.save.assert_called_once_with()
with mock.patch('nova.utils.spawn_on') as mock_spawn:
with mock.patch('nova.thread_pool_factory.spawn_on') as mock_spawn:
for _ in (1, 2, 3):
_do_it()
self.assertEqual(3, mock_spawn.call_count)
@@ -11915,16 +11898,20 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase,
def test_max_concurrent_live_semaphore_limited(self):
self.flags(max_concurrent_live_migrations=123)
mgr = manager.ComputeManager()
self.assertEqual(123, mgr._live_migration_executor._max_workers)
manager.ComputeManager()
executor = thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.LIVE_MIGRATION)
self.assertEqual(123, executor._max_workers)
def test_max_concurrent_live_semaphore_unlimited(self):
self.flags(max_concurrent_live_migrations=0)
mgr = manager.ComputeManager()
manager.ComputeManager()
executor = thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.LIVE_MIGRATION)
if utils.concurrency_mode_threading():
self.assertEqual(5, mgr._live_migration_executor._max_workers)
self.assertEqual(5, executor._max_workers)
else:
self.assertEqual(1000, mgr._live_migration_executor._max_workers)
self.assertEqual(1000, executor._max_workers)
@mock.patch('nova.objects.InstanceGroup.get_by_instance_uuid', mock.Mock(
side_effect=exception.InstanceGroupNotFound(group_uuid='')))
@@ -12444,7 +12431,7 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase,
def test_live_migration_submit_failed(self, mock_notify, mock_exc):
migration = objects.Migration(self.context, uuid=uuids.migration)
migration.save = mock.MagicMock()
with mock.patch('nova.utils.spawn_on') as mock_spawn:
with mock.patch('nova.thread_pool_factory.spawn_on') as mock_spawn:
mock_spawn.side_effect = RuntimeError
self.assertRaises(exception.LiveMigrationNotSubmitted,
self.compute.live_migration, self.context,
@@ -12464,7 +12451,7 @@ class ComputeManagerMigrationTestCase(test.NoDBTestCase,
mock.patch.object(
self.compute, '_record_task_end',
wraps=self.compute._record_task_end),
mock.patch('nova.utils.spawn_on'),
mock.patch('nova.thread_pool_factory.spawn_on'),
) as (mock_task_start, mock_task_end, mock_spawn):
mock_spawn.side_effect = RuntimeError
self.assertRaises(exception.LiveMigrationNotSubmitted,
+11 -5
View File
@@ -29,6 +29,7 @@ from nova import exception
from nova import objects
from nova import service_auth
from nova import test
from nova import thread_pool_factory
from nova import utils
@@ -480,7 +481,9 @@ class ContextTestCase(test.NoDBTestCase):
# the leaked thread check at the test case cleanup
def cleanup():
work.release()
utils.SCATTER_GATHER_EXECUTOR.shutdown(wait=True)
thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.SCATTER_GATHER).shutdown(
wait=True)
self.addCleanup(cleanup)
@@ -510,7 +513,10 @@ class ContextTestCase(test.NoDBTestCase):
def test_scatter_gather_cells_queued_task_cancelled(self, mock_warning):
# ensure that only one task can run at a time so we can simulate
# queued tasks
utils.SCATTER_GATHER_EXECUTOR = utils.create_executor(max_workers=1)
executor = thread_pool_factory.FACTORY._new_executor(
1, thread_pool_factory.ExecutorType.SCATTER_GATHER.value)
thread_pool_factory.FACTORY._all_executors[
thread_pool_factory.ExecutorType.SCATTER_GATHER] = executor
work = threading.Event()
@@ -519,7 +525,7 @@ class ContextTestCase(test.NoDBTestCase):
# cleanup
def cleanup():
work.set()
utils.SCATTER_GATHER_EXECUTOR.shutdown(wait=True)
executor.shutdown(wait=True)
self.addCleanup(cleanup)
@@ -548,9 +554,9 @@ class ContextTestCase(test.NoDBTestCase):
# let the started task eventually finish so the thread leak check at
# the test case cleanup is satisfied.
work.set()
utils.SCATTER_GATHER_EXECUTOR.shutdown(wait=True)
executor.shutdown(wait=True)
stats = utils.SCATTER_GATHER_EXECUTOR.statistics
stats = executor.statistics
# The task that wasn't started is cancelled when the scatter-gather
# timed out.
self.assertEqual(1, stats.cancelled)
+6 -5
View File
@@ -44,6 +44,7 @@ from nova.objects import service as service_obj
from nova import test
from nova.tests import fixtures
from nova.tests.unit import fake_instance
from nova import thread_pool_factory
from nova import utils
CONF = cfg.CONF
@@ -296,27 +297,27 @@ class TestIndirectionAPIFixture(testtools.TestCase):
class TestSpawnIsSynchronousFixture(testtools.TestCase):
def test_spawn_patch(self):
orig_spawn = utils.spawn
orig_spawn = thread_pool_factory.spawn
fix = fixtures.SpawnIsSynchronousFixture()
self.useFixture(fix)
self.assertNotEqual(orig_spawn, utils.spawn)
self.assertNotEqual(orig_spawn, thread_pool_factory.spawn)
def test_spawn_passes_through(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
tester = mock.MagicMock()
utils.spawn(tester.function, 'foo', bar='bar')
thread_pool_factory.spawn(tester.function, 'foo', bar='bar')
tester.function.assert_called_once_with('foo', bar='bar')
def test_spawn_return_has_result(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
future = utils.spawn(lambda x: '%s' % x, 'foo')
future = thread_pool_factory.spawn(lambda x: '%s' % x, 'foo')
foo = future.result()
self.assertEqual('foo', foo)
def test_spawn_callback(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
future = utils.spawn(mock.MagicMock)
future = thread_pool_factory.spawn(mock.MagicMock)
call_count = []
def fake(thread):
+1 -1
View File
@@ -447,7 +447,7 @@ class HackingTestCase(test.NoDBTestCase):
self._assert_has_errors(code, checks.check_greenthread_spawns,
expected_errors=errors)
code = "nova.utils.spawn(func, arg1, kwarg1=kwarg1)"
code = "nova.thread_pool_factory.spawn(func, arg1, kwarg1=kwarg1)"
self._assert_has_no_errors(code, checks.check_greenthread_spawns)
def test_config_option_regex_match(self):
File diff suppressed because it is too large. Load diff
-854
View File
@@ -13,7 +13,6 @@
# under the License.
import datetime
import futurist
import hashlib
import os
import os.path
@@ -28,13 +27,10 @@ from keystoneauth1 import session as ks_session
import netaddr
from openstack import exceptions as sdk_exc
from oslo_config import cfg
from oslo_context import context as common_context
from oslo_context import fixture as context_fixture
import oslo_service.backend as oslo_backend
from oslo_utils import encodeutils
from oslo_utils import fixture as utils_fixture
from nova import context
from nova import exception
from nova import monkey_patch
from nova.objects import base as obj_base
@@ -765,66 +761,6 @@ class SafeTruncateTestCase(test.NoDBTestCase):
self.assertEqual(254, len(byte_message))
class SpawnTestCase(test.NoDBTestCase):
def setUp(self):
super(SpawnTestCase, self).setUp()
self.useFixture(context_fixture.ClearRequestContext())
def test_spawn_no_context(self):
self.assertIsNone(common_context.get_current())
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual('test', args[0])
def fake(arg):
pass
executor = utils._get_default_executor()
with mock.patch.object(executor, "submit", _fake_spawn):
getattr(utils, "spawn")(fake, 'test')
self.assertIsNone(common_context.get_current())
def test_spawn_context(self):
self.assertIsNone(common_context.get_current())
ctxt = context.RequestContext('user', 'project')
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual(ctxt, args[0])
self.assertEqual('test', kwargs['kwarg1'])
def fake(context, kwarg1=None):
pass
executor = utils._get_default_executor()
with mock.patch.object(executor, "submit", _fake_spawn):
getattr(utils, "spawn")(fake, ctxt, kwarg1='test')
self.assertEqual(ctxt, common_context.get_current())
def test_spawn_context_different_from_passed(self):
self.assertIsNone(common_context.get_current())
ctxt = context.RequestContext('user', 'project')
ctxt_passed = context.RequestContext('user', 'project',
overwrite=False)
self.assertEqual(ctxt, common_context.get_current())
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual(ctxt_passed, args[0])
self.assertEqual('test', kwargs['kwarg1'])
def fake(context, kwarg1=None):
pass
executor = utils._get_default_executor()
with mock.patch.object(executor, "submit", _fake_spawn):
getattr(utils, "spawn")(fake, ctxt_passed, kwarg1='test')
self.assertEqual(ctxt, common_context.get_current())
class TestObjectCallHelpers(test.NoDBTestCase):
def test_with_primitives(self):
tester = mock.Mock()
@@ -1401,290 +1337,6 @@ class LatchErrorOnRaiseTests(test.NoDBTestCase):
self.assertIs(first, second)
class ScatterGatherExecutorTestCase(test.NoDBTestCase):
def test_executor_is_named(self):
executor = utils.get_scatter_gather_executor()
# NOTE(gibi): The executor is name both in normal run and in the test
# env. During testing we use a test-case-specific name, outside
# of test we use process name specific name instead. The test case
# specific name is added to help troubleshooting leaked executors
# between test case.
self.assertRegex(executor.name,
"nova.tests.unit.test_utils.ScatterGatherExecutor.*"
"test_executor_is_named.cell_worker")
def test_executor_type_eventlet(self):
if utils.concurrency_mode_threading():
self.skipTest("This test can only be run in eventlet mode.")
executor = utils.get_scatter_gather_executor()
self.assertEqual('GreenThreadPoolExecutor', type(executor).__name__)
@mock.patch.object(
utils, 'concurrency_mode_threading', new=mock.Mock(return_value=True))
def test_executor_type_and_size_threading(self):
self.flags(cell_worker_thread_pool_size=13)
executor = utils.get_scatter_gather_executor()
self.assertEqual('ThreadPoolExecutor', type(executor).__name__)
self.assertEqual(13, executor._max_workers)
def test_executor_destroy(self):
executor = utils.get_scatter_gather_executor()
self.assertIsNotNone(utils.SCATTER_GATHER_EXECUTOR)
utils.destroy_scatter_gather_executor()
self.assertIsNone(utils.SCATTER_GATHER_EXECUTOR)
self.assertFalse(executor.alive)
class DefaultExecutorTestCase(test.NoDBTestCase):
def test_executor_is_named(self):
executor = utils._get_default_executor()
# NOTE(gibi): The executor is name both in normal run and in the test
# env. During testing we use a test-case-specific name, outside
# of test we use process name specific name instead. The test case
# specific name is added to help troubleshooting leaked executors
# between test case.
self.assertRegex(executor.name,
"nova.tests.unit.test_utils.DefaultExecutor.*"
"test_executor_is_named.default")
def test_executor_type_and_size_eventlet(self):
if utils.concurrency_mode_threading():
self.skipTest("This test can only be run in eventlet mode.")
self.flags(default_green_pool_size=113)
executor = utils._get_default_executor()
self.assertEqual('GreenThreadPoolExecutor', type(executor).__name__)
self.assertEqual(113, executor._max_workers)
@mock.patch.object(
utils, 'concurrency_mode_threading', new=mock.Mock(return_value=True))
def test_executor_type_and_size_threading(self):
self.flags(default_thread_pool_size=13)
executor = utils._get_default_executor()
self.assertEqual('ThreadPoolExecutor', type(executor).__name__)
self.assertEqual(13, executor._max_workers)
def test_executor_destroy(self):
executor = utils._get_default_executor()
self.assertIsNotNone(utils.DEFAULT_EXECUTOR)
utils.destroy_default_executor()
self.assertIsNone(utils.DEFAULT_EXECUTOR)
self.assertFalse(executor.alive)
class SpawnOnTestCase(test.NoDBTestCase):
def test_spawn_on_submits_work(self):
executor = utils.get_scatter_gather_executor()
task = mock.MagicMock()
future = utils.spawn_on(executor, task, 13, foo='bar')
future.result()
task.assert_called_once_with(13, foo='bar')
@mock.patch.object(
utils, 'concurrency_mode_threading', new=mock.Mock(return_value=True))
@mock.patch.object(utils.LOG, 'warning')
def test_spawn_on_warns_on_full_executor(self, mock_warning):
# Ensure we have executor for a single task only at a time
self.flags(cell_worker_thread_pool_size=1)
executor = utils.get_scatter_gather_executor()
work = threading.Event()
started = threading.Event()
# let the blocked tasks finish after the test case so that the leaked
# thread check is not triggered during cleanup
self.addCleanup(work.set)
def task():
started.set()
work.wait()
# Start two tasks that will wait, the first will execute the second
# will wait in the queue
utils.spawn_on(executor, task)
utils.spawn_on(executor, task)
# wait for the first task to consume the single executor thread
started.wait()
# start one more task to trigger the fullness check.
utils.spawn_on(executor, task)
# We expect that spawn_on will warn due to the second task being is
# waiting in the queue, and no idle worker thread exists.
mock_warning.assert_called_once_with(
'The %s pool does not have free threads so the task %s will be '
'queued. If this happens repeatedly then the size of the pool is '
'too small for the load or there are stuck threads filling the '
'pool.',
'nova.tests.unit.test_utils.SpawnOnTestCase.'
'test_spawn_on_warns_on_full_executor.cell_worker', task)
@mock.patch.object(
utils, 'concurrency_mode_threading', new=mock.Mock(return_value=True))
@mock.patch.object(utils.LOG, 'warning')
def test_spawn_on_warns_on_full_executor_noname(self, mock_warning):
# Ensure we have executor for a single task only at a time
executor = utils.create_executor(max_workers=1)
work = threading.Event()
started = threading.Event()
# let the blocked tasks finish after the test case so that the leaked
# thread check is not triggered during cleanup
self.addCleanup(work.set)
def task():
started.set()
work.wait()
# Start two tasks that will wait, the first will execute the second
# will wait in the queue
utils.spawn_on(executor, task)
utils.spawn_on(executor, task)
# wait for the first task to consume the single executor thread
started.wait()
# start one more task to trigger the fullness check.
utils.spawn_on(executor, task)
# We expect that spawn_on will warn due to the second task being is
# waiting in the queue, and no idle worker thread exists.
mock_warning.assert_called_once_with(
'The %s pool does not have free threads so the task %s will be '
'queued. If this happens repeatedly then the size of the pool is '
'too small for the load or there are stuck threads filling the '
'pool.', 'unknown', task)
class ExecutorStatsTestCase(test.NoDBTestCase):
def setUp(self):
super().setUp()
self.work = threading.Event()
def _task_finishes(self):
return
def _task_fails(self):
raise ValueError()
def _task_running(self):
self.work.wait()
@mock.patch.object(
utils, 'concurrency_mode_threading', new=mock.Mock(return_value=False))
@mock.patch.object(utils.LOG, 'debug')
def test_stats_logged_eventlet(self, mock_debug):
env = os.environ.get('OS_NOVA_DISABLE_EVENTLET_PATCHING', '').lower()
if env in ('1', 'true', 'yes'):
self.skipTest(
"In native threading mode this case is covered by "
"test_stats_logged_threading")
# ensure that each task submission triggers stats printing
self.flags(thread_pool_statistic_period=0)
utils.spawn(self._task_finishes).result()
utils.spawn(self._task_fails).exception()
running = utils.spawn(self._task_running)
# avoid having a hanging thread leaking from the test case
def cleanup():
self.work.set()
running.result()
self.addCleanup(cleanup)
# The stats are printed *before* the work is submitted so we need an
# extra task submitted to get the stats from the above task.
utils.spawn(self._task_finishes).result()
print(mock_debug.mock_calls)
args = mock_debug.mock_calls[3][1]
self.assertEqual(
('State of %s GreenThreadPoolExecutor when submitting a new task: '
'workers: %d, max_workers: %d, work queued length: %d, stats: %s',
'nova.tests.unit.test_utils.ExecutorStatsTestCase.'
'test_stats_logged_eventlet.default', 1, 1000, 0),
args[0:5])
stats = args[5]
self.assertEqual(1, stats.failures)
self.assertEqual(2, stats.executed)
self.assertEqual(0, stats.cancelled)
@mock.patch.object(
utils, 'concurrency_mode_threading', new=mock.Mock(return_value=True))
@mock.patch.object(utils.LOG, 'debug')
def test_stats_logged_threading(self, mock_debug):
if not utils.concurrency_mode_threading():
self.skipTest(
"In eventlet mode this case is covered by "
"test_stats_logged_eventlet")
# ensure that each task submission triggers stats printing
self.flags(thread_pool_statistic_period=0)
# make the tasks sequential to help simulating queued task
self.flags(default_thread_pool_size=1)
utils.spawn(self._task_finishes).result()
utils.spawn(self._task_fails).exception()
running = utils.spawn(self._task_running)
# avoid having a hanging thread leaking from the test case
def cleanup():
self.work.set()
running.result()
self.addCleanup(cleanup)
# this will be queued as the only worker thread is held up by the
# running task
utils.spawn(self._task_finishes)
# this is also queued so we can cancel it to dequeue it
utils.spawn(self._task_finishes).cancel()
# The stats are printed *before* the work is submitted so we need an
# extra task submitted to get the stats from the above task.
utils.spawn(self._task_finishes)
args = mock_debug.mock_calls[5][1]
self.assertEqual(
('State of %s ThreadPoolExecutor when submitting a new task: '
'max_workers: %d, workers: %d, idle workers: %d, queued work: %d,'
' stats: %s',
'nova.tests.unit.test_utils.ExecutorStatsTestCase.'
'test_stats_logged_threading.default',
1, 1, 1, 3),
args[0:6])
stats = args[6]
self.assertEqual(1, stats.failures)
self.assertEqual(2, stats.executed)
self.assertEqual(1, stats.cancelled)
@mock.patch.object(utils.LOG, 'debug')
def test_stats_skipped_if_too_frequent(self, mock_debug):
self.flags(thread_pool_statistic_period=10)
utils.spawn(self._task_finishes).result()
mock_debug.assert_called()
mock_debug.reset_mock()
utils.spawn(self._task_finishes).result()
mock_debug.assert_not_called()
@mock.patch.object(utils.LOG, 'debug')
def test_stats_skipped_disabled(self, mock_info):
self.flags(thread_pool_statistic_period=-1)
utils.spawn(self._task_finishes).result()
mock_info.assert_not_called()
class OsloServiceBackendSelectionTestCase(test.NoDBTestCase):
def setUp(self):
# NOTE(gibi): We need this as the base test class would trigger
@@ -2127,509 +1779,3 @@ class TestFairLockGuard(test.NoDBTestCase):
self.assertFalse(lock_guard.is_locked())
mock_fair_locks.get.assert_not_called()
class StaticallyDelayingCancellableTaskExecutorWrapperTest(test.NoDBTestCase):
def test_submit_one(self):
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.01, utils._get_default_executor())
self.addCleanup(executor.shutdown)
task_done = threading.Event()
def task(num, foo):
self.assertEqual(12, num)
self.assertEqual("bar", foo)
task_done.set()
return foo + str(num)
future = executor.submit_with_delay(task, 12, foo="bar")
result = future.result()
self.assertTrue(task_done.is_set())
self.assertEqual("bar12", result)
def test_submit_one_exception_result(self):
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.01, utils._get_default_executor())
self.addCleanup(executor.shutdown)
task_done = threading.Event()
exc_to_raise = ValueError()
def task(num, foo):
self.assertEqual(12, num)
self.assertEqual("bar", foo)
task_done.set()
raise exc_to_raise
future = executor.submit_with_delay(task, 12, foo="bar")
exc = future.exception()
self.assertTrue(task_done.is_set())
self.assertEqual(exc_to_raise, exc)
def test_submit_two_non_overlapping(self):
def task1():
return 42
def task2():
return 13
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.01, utils._get_default_executor())
self.addCleanup(executor.shutdown, wait=True)
future1 = executor.submit_with_delay(task1)
self.assertEqual(42, future1.result())
future2 = executor.submit_with_delay(task2)
self.assertEqual(13, future2.result())
self.assertTrue(executor._queue.empty())
def test_submit_second_while_delaying_first(self):
task1_started = threading.Event()
def task1():
task1_started.set()
return 42
task2_started = threading.Event()
def task2():
task2_started.set()
return 13
# Create a "long" delay so the task will be actively managed by
# the wrapper while we submit the second task
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2, utils._get_default_executor())
self.addCleanup(executor.shutdown, wait=True)
task1_start = time.monotonic()
future1 = executor.submit_with_delay(task1)
# wait a bit so the wrapper is picking it up and waiting for its
# deadline
time.sleep(1)
self.assertTrue(executor._queue.empty())
self.assertFalse(task1_started.is_set())
# now submit the second task, it will be queued
task2_start = time.monotonic()
future2 = executor.submit_with_delay(task2)
self.assertFalse(executor._queue.empty())
self.assertFalse(task1_started.is_set())
# eventually both tasks finishes
self.assertEqual(42, future1.result())
task1_end = time.monotonic()
self.assertEqual(13, future2.result())
task2_end = time.monotonic()
# and both tasks took about delay seconds individually, but the two
# tasks together took less than 2x delay seconds as they were
# overlapped.
task1_runtime = task1_end - task1_start
self.assertLess(task1_runtime, 2.5)
self.assertGreater(task1_runtime, 2.0)
task2_runtime = task2_end - task2_start
self.assertLess(task2_runtime, 2.5)
self.assertGreater(task2_runtime, 2.0)
total_runtime = task2_end - task1_start
self.assertLess(total_runtime, 4)
def test_submit_multiple(self):
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.1, utils._get_default_executor())
def task(i):
return 2 * i
futures = []
for i in range(20):
futures.append(executor.submit_with_delay(task, i))
for i, f in enumerate(futures):
self.assertEqual(2 * i, f.result())
executor.shutdown(wait=True)
def test_submit_multiple_executor_rejects_first_executes_second(self):
def task1():
return 42
def task2():
return 13
check_and_reject = mock.Mock(
side_effect=[futurist.RejectedSubmission(), None])
if utils.concurrency_mode_threading():
ex = futurist.ThreadPoolExecutor(
max_workers=1, check_and_reject=check_and_reject)
else:
ex = futurist.GreenThreadPoolExecutor(
max_workers=1, check_and_reject=check_and_reject)
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.1, ex)
self.addCleanup(executor.shutdown, wait=True)
future1 = executor.submit_with_delay(task1)
future2 = executor.submit_with_delay(task2)
self.assertEqual(
futurist.RejectedSubmission, type(future1.exception()))
self.assertEqual(13, future2.result())
def test_cancel_during_delay(self):
task1_started = threading.Event()
def task1():
task1_started.set()
return 42
def task2():
return 13
# Create a "long" delay so the task will be actively delayed by
# the wrapper when we cancel it
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2, utils._get_default_executor())
self.addCleanup(executor.shutdown, wait=True)
future1 = executor.submit_with_delay(task1)
# wait a bit to let the task being picked up
time.sleep(1)
# it is not in the queue, so it is picked up
self.assertTrue(executor._queue.empty())
# but not executing yet so it is being delayed
self.assertFalse(task1_started.is_set())
# cancel the task
future1.cancel()
# Submit and wait for the execution of the second task to prove
# that the executor had time to finish waiting for the deadline of
# the first task, detected the cancellation and skipped the task,
# then executed the second task.
future2 = executor.submit_with_delay(task2)
self.assertEqual(13, future2.result())
# task1 is still not executed
self.assertFalse(task1_started.is_set())
self.assertTrue(future1.cancelled())
# no tasks remaining in the executor queue
self.assertTrue(executor._queue.empty())
def test_cancel_while_in_queue(self):
task1_started = threading.Event()
def task1():
task1_started.set()
return 42
task2_started = threading.Event()
def task2():
task2_started.set()
return 13
# Create a "long" delay so one task will be actively delayed while
# we submit a second task then cancel the second task.
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2, utils._get_default_executor())
future1 = executor.submit_with_delay(task1)
# Wait a bit to let the task being picked up.
time.sleep(1)
# It is not in the queue, so it is picked up,
self.assertTrue(executor._queue.empty())
# but not executing yet, so it is being delayed
self.assertFalse(task1_started.is_set())
# Submit a second task that will be queued.
future2 = executor.submit_with_delay(task2)
self.assertFalse(executor._queue.empty())
self.assertFalse(task2_started.is_set())
# Cancel the second task while it is in the queue.
future2.cancel()
# The first task should finish normally
self.assertEqual(42, future1.result())
# But the second task should never be executed.
# To prove that we shutdown both the wrapper and the real executor
# then check the second task again.
executor.shutdown(wait=True)
utils._get_default_executor().shutdown(wait=True)
self.assertFalse(task2_started.is_set())
self.assertTrue(future2.cancelled())
def test_instantaneous_shutdown(self):
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.1, utils._get_default_executor())
self.addCleanup(executor.shutdown, wait=True)
executor.shutdown(wait=False)
self.assertTrue(executor._shutdown)
def test_shutdown_wait(self):
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.1, utils._get_default_executor())
executor.shutdown(wait=True)
self.assertTrue(executor._shutdown)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
# Shutting down the wrapper does not affect the real executor
self.assertTrue(executor._executor.alive)
def test_submit_after_shutdown_rejected(self):
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.1, utils._get_default_executor())
executor.shutdown()
self.assertTrue(executor._shutdown)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
exc = self.assertRaises(
RuntimeError, executor.submit_with_delay, lambda: None)
self.assertEqual(
"Cannot schedule new tasks after being shutdown", str(exc))
def test_submit_while_shutting_down(self):
task_started = threading.Event()
def task():
task_started.set()
# Create a "long" delay so the task will be actively managed by
# the wrapper while the test calls shutdown on it.
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2, utils._get_default_executor())
self.addCleanup(executor.shutdown, wait=True)
executor.submit_with_delay(task)
executor.shutdown(wait=False)
# the task is actively managed, delayed, by our wrapper while we are
# shutting the executor down, we should not be able to add new tasks
# even if the shutdown is not finished yet
self.assertFalse(task_started.is_set())
exc = self.assertRaises(
RuntimeError, executor.submit_with_delay, lambda: None)
self.assertEqual(
"Cannot schedule new tasks after being shutdown", str(exc))
def test_shutdown_after_task_finished(self):
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
0.01, utils._get_default_executor())
self.addCleanup(executor.shutdown)
def task():
return
future = executor.submit_with_delay(task)
future.result()
executor.shutdown()
self.assertTrue(executor._shutdown)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
def test_no_wait_shutdown_task_finishes_normally(self):
task_started = threading.Event()
def task():
task_started.set()
return 42
# Create a "long" delay so the task will be actively managed by
# the wrapper while the test calls shutdown on it.
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2, utils._get_default_executor())
self.addCleanup(executor.shutdown, wait=True)
future = executor.submit_with_delay(task)
# Task is not executing it is being delayed
self.assertFalse(task_started.is_set())
# We expect that shutdown returns even though there are tasks being
# actively managed, delayed, by the executor as we called it with
# wait=False
executor.shutdown(wait=False)
self.assertTrue(executor._shutdown)
self.assertFalse(executor._queue.empty())
self.assertTrue(executor.is_alive)
# Task is still delayed
self.assertFalse(task_started.is_set())
# and it is not cancelled
self.assertFalse(future.cancelled())
# and eventually executed
self.assertEqual(42, future.result())
# and eventually the executor is terminated. This also covers the case
# when multiple shutdown call is made to the same executor.
executor.shutdown(wait=True)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
def test_no_wait_shutdown_multiple_tasks_finishes_normally(self):
task1_started = threading.Event()
def task1():
task1_started.set()
return 42
task2_started = threading.Event()
def task2():
task2_started.set()
return 13
# Create a "long" delay so the task will be actively managed by
# the wrapper while the test calls shutdown on it.
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2, utils._get_default_executor())
future1 = executor.submit_with_delay(task1)
future2 = executor.submit_with_delay(task2)
# Tasks are not executing they are being delayed
self.assertFalse(task1_started.is_set())
self.assertFalse(task2_started.is_set())
# wait a bit so the wrapper actually starts waiting for the deadline
# of task1
time.sleep(1)
# Task are still not executing
self.assertFalse(task1_started.is_set())
self.assertFalse(task2_started.is_set())
# task1 is already popped from the queue and the code is waiting for
# its deadline, while task2 is in the queue waiting
self.assertEqual(1, executor._queue.qsize())
# Shut down the executor. We expect that the tasks are still executed
executor.shutdown(wait=True)
self.assertEqual(42, future1.result())
self.assertEqual(13, future2.result())
# and our wrapper is in a shutdown state
self.assertTrue(executor._shutdown)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
def test_shutdown_wait_task_finishes_normally(self):
task_started = threading.Event()
def task():
task_started.set()
return 42
# Create a "long" delay so the task will be actively managed by
# the wrapper while the test calls shutdown on it.
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2, utils._get_default_executor())
future = executor.submit_with_delay(task)
# Task is not executing it is being delayed
self.assertFalse(task_started.is_set())
# We expect that shutdown waits for the task to be submitted to the
# real executor after the delay
executor.shutdown(wait=True)
# Task is now submitted to the real executor so it can actually run
# and produce a result
result = future.result()
self.assertTrue(task_started.is_set())
self.assertEqual(42, result)
# but our wrapper is in a shutdown state
self.assertTrue(executor._shutdown)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
def test_shutdown_does_not_wait_for_cancelled_task(self):
task_started = threading.Event()
def task():
task_started.set()
return 42
# Create a very long delay so the task will be actively managed by
# the wrapper while the test calls shutdown on it and we can
# check that the wrapper detects cancelled tasks during shutdown and
# does not wait for the whole deadline.
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2000, utils._get_default_executor())
future = executor.submit_with_delay(task)
# Task is not executing it is being delayed
self.assertFalse(task_started.is_set())
# Cancel the task, shutdown the executor. We expect that the cancelled
# task is not submitted for execution.
future.cancel()
executor.shutdown(wait=True)
self.assertFalse(task_started.is_set())
self.assertTrue(future.cancelled())
# and our wrapper is in a shutdown state
self.assertTrue(executor._shutdown)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
def test_shutdown_does_not_wait_for_multiple_cancelled_tasks(self):
task1_started = threading.Event()
def task1():
task1_started.set()
return 42
task2_started = threading.Event()
def task2():
task2_started.set()
return 13
# Create a very long delay so the task will be actively managed by
# the wrapper while the test calls shutdown on it and we can
# check that the wrapper detects cancelled tasks during shutdown and
# does not wait for the whole deadline.
executor = utils.StaticallyDelayingCancellableTaskExecutorWrapper(
2000, utils._get_default_executor())
future1 = executor.submit_with_delay(task1)
future2 = executor.submit_with_delay(task2)
# Tasks are not executing they are being delayed
self.assertFalse(task1_started.is_set())
self.assertFalse(task2_started.is_set())
# wait a bit so the wrapper actually starts waiting for the deadline
# of task1
time.sleep(1)
# Task are still not executing
self.assertFalse(task1_started.is_set())
self.assertFalse(task2_started.is_set())
# task1 is already popped from the queue and the code is waiting for
# its deadline, while task2 is in the queue waiting
self.assertEqual(1, executor._queue.qsize())
# Cancel both tasks then shutdown the executor and expect that no
# task will be executed.
future1.cancel()
future2.cancel()
executor.shutdown(wait=True)
self.assertFalse(task1_started.is_set())
self.assertTrue(future1.cancelled())
self.assertFalse(task2_started.is_set())
self.assertTrue(future2.cancelled())
# and our wrapper is in a shutdown state
self.assertTrue(executor._shutdown)
self.assertTrue(executor._queue.empty())
self.assertFalse(executor.is_alive)
+3 -2
View File
@@ -21,6 +21,7 @@ from unittest import mock
import fixtures
from nova import test
from nova import thread_pool_factory
from nova import utils
from nova.virt.disk.mount import nbd
from nova.virt.image import model as imgmodel
@@ -324,8 +325,8 @@ class NbdTestCase(test.NoDBTestCase):
n.get_dev()
chosen_devices.append(n.device)
thread1 = utils.spawn(get_a_device)
thread2 = utils.spawn(get_a_device)
thread1 = thread_pool_factory.spawn(get_a_device)
thread2 = thread_pool_factory.spawn(get_a_device)
thread1.result()
thread2.result()
+12 -11
View File
@@ -94,6 +94,7 @@ from nova.tests.unit.objects import test_instance_numa
from nova.tests.unit.objects import test_pci_device
from nova.tests.unit.objects import test_vcpu_model
from nova.tests.unit import utils as test_utils
from nova import thread_pool_factory
from nova import utils
from nova import version
from nova.virt import block_device as driver_block_device
@@ -566,8 +567,8 @@ class CacheConcurrencyTestCase(test.NoDBTestCase):
wait1 = threading.Event()
done1 = threading.Event()
sig1 = threading.Event()
thr1 = utils.spawn(backend.by_name(self._fake_instance(uuid),
'name').cache,
thr1 = thread_pool_factory.spawn(
backend.by_name(self._fake_instance(uuid), 'name').cache,
_concurrency, 'fname', None,
signal=sig1, wait=wait1, done=done1)
utils.cooperative_yield()
@@ -577,8 +578,8 @@ class CacheConcurrencyTestCase(test.NoDBTestCase):
wait2 = threading.Event()
done2 = threading.Event()
sig2 = threading.Event()
thr2 = utils.spawn(backend.by_name(self._fake_instance(uuid),
'name').cache,
thr2 = thread_pool_factory.spawn(
backend.by_name(self._fake_instance(uuid), 'name').cache,
_concurrency, 'fname', None,
signal=sig2, wait=wait2, done=done2)
@@ -604,8 +605,8 @@ class CacheConcurrencyTestCase(test.NoDBTestCase):
wait1 = threading.Event()
done1 = threading.Event()
sig1 = threading.Event()
thr1 = utils.spawn(backend.by_name(self._fake_instance(uuid),
'name').cache,
thr1 = thread_pool_factory.spawn(
backend.by_name(self._fake_instance(uuid), 'name').cache,
_concurrency, 'fname2', None,
signal=sig1, wait=wait1, done=done1)
utils.cooperative_yield()
@@ -615,8 +616,8 @@ class CacheConcurrencyTestCase(test.NoDBTestCase):
wait2 = threading.Event()
done2 = threading.Event()
sig2 = threading.Event()
thr2 = utils.spawn(backend.by_name(self._fake_instance(uuid),
'name').cache,
thr2 = thread_pool_factory.spawn(
backend.by_name(self._fake_instance(uuid), 'name').cache,
_concurrency, 'fname1', None,
signal=sig2, wait=wait2, done=done2)
utils.cooperative_yield()
@@ -13763,7 +13764,7 @@ class LibvirtConnTestCase(test.NoDBTestCase,
drvr._live_migration_uri(target_connection),
params=params, flags=0)
@mock.patch.object(utils, 'spawn')
@mock.patch.object(thread_pool_factory, 'spawn')
@mock.patch.object(host.Host, 'get_guest')
@mock.patch.object(fakelibvirt.Connection, '_mark_running')
@mock.patch.object(libvirt_driver.LibvirtDriver,
@@ -15665,7 +15666,7 @@ class LibvirtConnTestCase(test.NoDBTestCase,
expected_switch=True)
@mock.patch.object(host.Host, "get_connection")
@mock.patch.object(utils, "spawn")
@mock.patch.object(thread_pool_factory, "spawn")
@mock.patch.object(libvirt_driver.LibvirtDriver, "_live_migration_monitor")
@mock.patch.object(host.Host, "get_guest")
@mock.patch.object(fakelibvirt.Connection, "_mark_running")
@@ -15724,7 +15725,7 @@ class LibvirtConnTestCase(test.NoDBTestCase,
self._test_live_migration_main(mon_side_effect=Exception)
@mock.patch.object(host.Host, "get_connection", new=mock.Mock())
@mock.patch.object(utils, "spawn")
@mock.patch.object(thread_pool_factory, "spawn")
@mock.patch.object(host.Host, "get_guest")
@mock.patch.object(
libvirt_driver.LibvirtDriver, "_live_migration_copy_disk_paths")
+27 -14
View File
@@ -35,6 +35,7 @@ from nova import test
from nova.tests import fixtures as nova_fixtures
from nova.tests.fixtures import libvirt as fakelibvirt
from nova.tests.fixtures import libvirt_data as fake_libvirt_data
from nova import thread_pool_factory
from nova import utils
from nova.virt import event
from nova.virt.libvirt import config as vconfig
@@ -82,8 +83,10 @@ class HostTestCase(test.NoDBTestCase):
# to cover code there. So we partially instantiate the event handling
# logic here and pumping the events in the test cases individually.
h._delayed_executor = (
utils.StaticallyDelayingCancellableTaskExecutorWrapper(
delay=15, executor=utils._get_default_executor()))
thread_pool_factory.
StaticallyDelayingCancellableTaskExecutorWrapper(
delay=15, executor=thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT)))
return h
@@ -130,8 +133,8 @@ class HostTestCase(test.NoDBTestCase):
self.assertEqual(0, len(log_mock.method_calls),
'LOG should not be used in _connect_auth_cb.')
@mock.patch.object(
utils, 'StaticallyDelayingCancellableTaskExecutorWrapper')
@mock.patch.object(thread_pool_factory,
'StaticallyDelayingCancellableTaskExecutorWrapper')
def test_event_dispatch(self, mock_wrapper):
# Validate that the libvirt self-pipe for forwarding
# events between threads is working sanely
@@ -180,7 +183,8 @@ class HostTestCase(test.NoDBTestCase):
self.assertEqual(want_events, got_events)
mock_wrapper.assert_called_once_with(
delay=15, executor=utils._get_default_executor())
delay=15, executor=thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT))
# STOPPED is delayed so it's handled separately
mock_wrapper.return_value.submit_with_delay(
hostimpl._event_emit, event4)
@@ -292,7 +296,9 @@ class HostTestCase(test.NoDBTestCase):
test.MatchType(libvirt_guest.Guest), instance=None,
logging_ok=False)
@mock.patch('nova.utils.StaticallyDelayingCancellableTaskExecutorWrapper')
@mock.patch(
'nova.thread_pool_factory.'
'StaticallyDelayingCancellableTaskExecutorWrapper')
def test_event_emit_delayed_call_delayed(self, mock_wrapper):
ev = event.LifecycleEvent(
"cef19ce0-0ca2-11df-855d-b19fbce37686",
@@ -303,11 +309,14 @@ class HostTestCase(test.NoDBTestCase):
hostimpl._event_emit_delayed(ev)
mock_wrapper.assert_called_once_with(
delay=15, executor=utils._get_default_executor())
delay=15, executor=thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT))
mock_wrapper.return_value.submit_with_delay.assert_called_once_with(
hostimpl._event_emit, ev)
@mock.patch('nova.utils.StaticallyDelayingCancellableTaskExecutorWrapper')
@mock.patch(
'nova.thread_pool_factory.'
'StaticallyDelayingCancellableTaskExecutorWrapper')
def test_event_emit_delayed_call_delayed_pending(self, mock_wrapper):
hostimpl = self._create_host(
'qemu:///system', lifecycle_event_handler=lambda e: None)
@@ -439,8 +448,8 @@ class HostTestCase(test.NoDBTestCase):
mock_event.side_effect = fake_register
# call concurrently
thr1 = utils.spawn(get_conn_currency, self.host)
thr2 = utils.spawn(get_conn_currency, self.host)
thr1 = thread_pool_factory.spawn(get_conn_currency, self.host)
thr2 = thread_pool_factory.spawn(get_conn_currency, self.host)
# let threads run
utils.cooperative_yield()
@@ -2118,8 +2127,10 @@ class TestLibvirtSEV(test.NoDBTestCase):
self.host = host.Host("qemu:///system")
self.addCleanup(self.host.cleanup)
self.host._delayed_executor = (
utils.StaticallyDelayingCancellableTaskExecutorWrapper(
delay=0.1, executor=utils._get_default_executor()))
thread_pool_factory.
StaticallyDelayingCancellableTaskExecutorWrapper(
delay=0.1, executor=thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT)))
self.kernel_sev = None
self.kernel_sev_es = None
@@ -2560,8 +2571,10 @@ class LibvirtTpoolProxyTestCase(test.NoDBTestCase):
self.useFixture(nova_fixtures.LibvirtFixture())
self.host = host.Host("qemu:///system")
self.host._delayed_executor = (
utils.StaticallyDelayingCancellableTaskExecutorWrapper(
delay=0.1, executor=utils._get_default_executor()))
thread_pool_factory.
StaticallyDelayingCancellableTaskExecutorWrapper(
delay=0.1, executor=thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT)))
self.addCleanup(self.host.cleanup)
def _stub_xml(uuid):
@@ -23,7 +23,7 @@ from oslo_utils.fixture import uuidsentinel as uuids
from nova import exception
from nova import test
from nova import utils
from nova import thread_pool_factory
from nova.virt.libvirt import config as libvirt_config
from nova.virt.libvirt import guest as libvirt_guest
from nova.virt.libvirt import host as libvirt_host
@@ -656,7 +656,7 @@ class MountManagerTestCase(test.NoDBTestCase):
# Call host_up in a separate thread because it will block, and give
# it plenty of time to race
host_up = utils.spawn(self.m.host_up, mock.sentinel.host)
host_up = thread_pool_factory.spawn(self.m.host_up, mock.sentinel.host)
time.sleep(0.01)
# Assert that we haven't instantiated a new state while there's an
+643
View File
@@ -0,0 +1,643 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import enum
import functools
import multiprocessing
import queue
import threading
import time
import typing as ty
import futurist
from oslo_context import context as common_context
from oslo_log import log as logging
from oslo_utils import importutils
import nova.conf
from nova import utils
profiler = importutils.try_import('osprofiler.profiler')
CONF = nova.conf.CONF
LOG = logging.getLogger(__name__)
# NOTE(gibi): futurist does not expose the common based class.
# NOTE(gibi): we can simplify this when eventlet is removed.
Executor: ty.TypeAlias = (
futurist.GreenThreadPoolExecutor | futurist.ThreadPoolExecutor)
def get_max_concurrent_builds():
if CONF.max_concurrent_builds > 0:
return CONF.max_concurrent_builds
# setting CONF.max_concurrent_builds to 0 (unlimited)
# is deprecated but still supported, so we need to use a sane
# default values for each threading mode
LOG.warning("Nova compute deprecated the support of unlimited "
"parallel instance builds so "
"[DEFAULT]max_concurrent_builds configured "
"with value 0 is deprecated and will not be supported "
"in future releases. Please set an explicit positive "
"value to this config option instead.")
if utils.concurrency_mode_threading():
# Fall back to the default of the config
return 10
else:
# In eventlet mode we need to keep backward compatibility, and
# we use 1000 to emulate unlimited
return 1000
def get_max_concurrent_snapshots():
if CONF.max_concurrent_snapshots > 0:
return CONF.max_concurrent_snapshots
# setting CONF.max_concurrent_snapshots to 0 (unlimited)
# is deprecated but still supported, so we need to use a sane
# default values for each threading mode
LOG.warning("Nova compute deprecated the support of unlimited "
"parallel instance snapshots so "
"[DEFAULT]max_concurrent_snapshots configured "
"with value 0 is deprecated and will not be supported "
"in future releases. Please set an explicit positive "
"value to this config option instead.")
if utils.concurrency_mode_threading():
# Fall back to the default of the config
return 5
else:
# In eventlet mode we need to keep backward compatibility, and
# we use 1000 to emulate unlimited
return 1000
def get_max_concurrent_live_migrations():
if CONF.max_concurrent_live_migrations > 0:
return CONF.max_concurrent_live_migrations
# setting CONF.max_concurrent_live_migrations to 0 (unlimited)
# is deprecated but still supported, so we need to use a sane
# default values for each threading mode
LOG.warning("Nova compute deprecated the support of unlimited "
"parallel live migration so "
"[DEFAULT]max_concurrent_live_migrations configured "
"with value 0 is deprecated and will not be supported "
"in future releases. Please set an explicit positive"
"value to this config option instead.")
if utils.concurrency_mode_threading():
return 5
else:
# In eventlet mode we need to keep backward compatibility and
# 1000 greenthreads to emulate unlimited
return 1000
class ExecutorType(enum.Enum):
"""The process wide singleton executors.
The enum value is used both as the executor's name suffix and
as the key which ThreadPoolFactory tracks it.
"""
DEFAULT = "default"
SCATTER_GATHER = "cell_worker"
CACHE_IMAGES = "cache_images"
LONG_TASK = "long_task"
SYNC_POWER = "sync_power_state"
LIVE_MIGRATION = "live_migration"
class ExecutorsPoolSize:
"""Sizing for every singleton executor."""
@staticmethod
def _default_pool_size():
return (
CONF.default_thread_pool_size if utils.concurrency_mode_threading()
else CONF.default_green_pool_size)
@staticmethod
def _scatter_gather_pool_size():
return (
CONF.cell_worker_thread_pool_size
if utils.concurrency_mode_threading() else 1000)
@staticmethod
def _cache_images_pool_size():
return CONF.image_cache.precache_concurrency
@staticmethod
def _sync_power_pool_size():
return CONF.sync_power_state_pool_size
@staticmethod
def _long_task_pool_size():
max_builds = get_max_concurrent_builds()
max_snapshots = get_max_concurrent_snapshots()
if utils.concurrency_mode_threading():
max_tasks = max(max_builds, max_snapshots)
if max_builds != max_snapshots:
LOG.warning(
"In native threading mode the number of concurrent "
"builds, and snapshots should be limited to the "
"same number. The current configuration has differing "
"limits: max_concurrent_builds: %d, "
"max_concurrent_snapshots: %d. "
"Nova will use a single, overall limit of %d for these "
"tasks.",
max_builds, max_snapshots, max_tasks)
else:
# In eventlet mode we use the individual build/snapshot
# semaphores to limit the concurrent tasks, so just size the
# pool to potentially host all of them.
return max_builds + max_snapshots
return max_tasks
_EXECUTOR_POOL_SIZE = {
ExecutorType.DEFAULT: _default_pool_size,
ExecutorType.SCATTER_GATHER: _scatter_gather_pool_size,
ExecutorType.CACHE_IMAGES: _cache_images_pool_size,
ExecutorType.LONG_TASK: _long_task_pool_size,
ExecutorType.SYNC_POWER: _sync_power_pool_size,
ExecutorType.LIVE_MIGRATION: get_max_concurrent_live_migrations,
}
@classmethod
def get(cls, executor_type):
return cls._EXECUTOR_POOL_SIZE[executor_type]()
class ThreadPoolFactory:
"""Lifecycle management factory of every thread pool executors.
This provides the single entry point for to create and get the thread
pool executors. It create the singleton executors and those are tracked
for central management.
Every executor created through this factory is tracked so that
we do not need to know every individual executors for cleanup/shutdown
and shutdown_all() can be used as a common way to shut down all of them.
"""
def __init__(self):
# NOTE(gmaan): This lock makes executor creation and shutdown mutual
# exclusive so that no new executor can be created once shutdown has
# started.
self._shutdown_lock = threading.Lock()
self._shutdown = False
self._all_executors = {}
def _new_executor(self, max_workers, name):
"""Create a new executor."""
if utils.concurrency_mode_threading():
executor = futurist.ThreadPoolExecutor(max_workers)
else:
executor = futurist.GreenThreadPoolExecutor(max_workers)
pname = multiprocessing.current_process().name
executor.name = f"{pname}.{name}"
return executor
def get_executor(self, executor_type):
"""Return the shared, process-wide singleton executor for
`executor_type`, lazily creating it the first time it is requested,
sized via ExecutorsPoolSize.
"""
executor = self._all_executors.get(executor_type)
if not executor:
with self._shutdown_lock:
if self._shutdown:
raise RuntimeError(
"Cannot create the %s thread pool executor because "
"shutdown has been started." %
executor_type.value)
executor = self._all_executors.get(executor_type)
if not executor:
max_workers = ExecutorsPoolSize.get(executor_type)
executor = self._new_executor(
max_workers, executor_type.value)
self._all_executors[executor_type] = executor
LOG.info(
"The %s thread pool %s is initialized",
executor_type.value, executor.name)
return executor
def _teardown_all(self, mark_shutdown=True, wait=False):
with self._shutdown_lock:
if mark_shutdown:
self._shutdown = True
executors = list(self._all_executors.values())
self._all_executors.clear()
for executor in executors:
name = getattr(executor, "name")
LOG.info("The thread pool %s is shutting down", name)
executor.shutdown(wait=wait)
LOG.info("The thread pool %s is shutdown", name)
def shutdown_all(self):
"""Shuts down every executor created via get_executor() so far.
Services create different sets of executors depending on what they
need. Since get_executor() registers every executor it creates,
this gives nova.service.Service's graceful shutdown flow a single,
common call to shut all of them down at the end.
NOTE: This permanently marks this factory as shutdown (set
self._shutdown=True), after which get_executor() will always raise.
Use this only for a process's real graceful shutdown (e.g. on
SIGTERM), never as a way to just reset/clear the executors. For
example, scheduler and conductor service use os.fork() to create
their worker instance which copies this flag into every worker,
so calling this on a process that will go on to fork workers will
permanently break executor creation in all of them. Use reset_all()
for that case instead.
"""
self._teardown_all()
def reset_all(self):
"""Reset/Clear every executor.
This shuts down every executor and clears self._all_executors but
does not set any flag/state on this factory. Also, it shutdown the
executors with wait=True so it gives executors to finish the things
already in queue. Those two are the difference from shutdown_all().
It is used before the main process uses os.fork() to create multiple
workers. Any executor created in the parent or this factory state
must not be inherited by the forked worker. Use this when you need
to clear the executors but do not want to mark this thread pool
factory as shutdown.
TODO(gmaan): Once we make our services to use spawn instead of fork
then we do not need this reset_all() or cleanup the threads before
child process/workers are created.
"""
self._teardown_all(mark_shutdown=False, wait=True)
FACTORY = ThreadPoolFactory()
def get_executor(executor_type):
return FACTORY.get_executor(executor_type)
def shutdown_all_executors():
"""Shutdown every executor of a process."""
FACTORY.shutdown_all()
def reset_all_executors():
"""Reset every executor of a process.
"""
FACTORY.reset_all()
def _serialize_profile_info():
if not profiler:
return None
prof = profiler.get()
trace_info = None
if prof:
# FIXME(DinaBelova): we'll add profiler.get_info() method
# to extract this info -> we'll need to update these lines
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
return trace_info
def _executor_is_full(executor: Executor) -> bool:
if utils.concurrency_mode_threading():
# TODO(gibi): Move this whole logic to futurist ThreadPoolExecutor
# so that we can avoid accessing the internals of the executor
with executor._shutdown_lock:
idle_workers = len(
[w for w in executor._workers if w.idle]) > 0
queued_tasks = executor._work_queue.qsize() > 0
return queued_tasks and not idle_workers
return False
def _log_executor_stats(executor: Executor, executor_type) -> None:
if CONF.thread_pool_statistic_period < 0:
return
last_stats = getattr(executor, "last_stats", None)
name = executor_type.value
allowed_stat_age = (
time.monotonic() - CONF.thread_pool_statistic_period)
if last_stats and last_stats > allowed_stat_age:
return
executor.last_stats = time.monotonic()
stats: futurist.ExecutorStatistics = executor.statistics
if isinstance(executor, futurist.ThreadPoolExecutor):
LOG.debug(
"State of %s ThreadPoolExecutor when submitting a new "
"task: max_workers: %d, workers: %d, idle workers: %d, "
"queued work: %d, stats: %s",
name,
executor._max_workers, len(executor._workers),
len([w for w in executor._workers if w.idle]),
executor._work_queue.qsize(), stats)
elif isinstance(executor, futurist.GreenThreadPoolExecutor):
LOG.debug(
"State of %s GreenThreadPoolExecutor when submitting a new "
"task: workers: %d, max_workers: %d, work queued length: "
"%d, stats: %s",
name,
len(executor._pool.coroutines_running), executor._pool.size,
executor._delayed_work.unfinished_tasks, stats)
def spawn_on(
executor_type: ExecutorType,
func: ty.Callable[..., ty.Any],
*args: ty.Any,
**kwargs: ty.Any,
) -> futurist.Future:
"""Passthrough method to run func on a thread in the named executor.
Gets (creating it on first use) the executor for `executor_type` and
runs func on it.
It will also grab the context from the threadlocal store and add it to
the store on the new thread. This allows for continuity in logging the
context when using this method to spawn a new thread.
"""
executor = get_executor(executor_type)
_log_executor_stats(executor, executor_type)
if _executor_is_full(executor):
LOG.warning(
"The %s pool does not have free threads so the task %s will be "
"queued. If this happens repeatedly then the size of the pool is "
"too small for the load or there are stuck threads filling the "
"pool.", getattr(executor, "name", "unknown"), func)
_context = common_context.get_current()
profiler_info = _serialize_profile_info()
@functools.wraps(func)
def context_wrapper(*args, **kwargs):
# NOTE: If update_store is not called after spawning a thread, it won't
# be available for the logger to pull from threadlocal storage.
if _context is not None:
_context.update_store()
if profiler_info and profiler:
profiler.init(**profiler_info)
return func(*args, **kwargs)
return executor.submit(context_wrapper, *args, **kwargs)
def spawn(
func: ty.Callable[..., ty.Any],
*args: ty.Any, **kwargs: ty.Any,
) -> futurist.Future:
"""Passthrough method for eventlet.spawn.
Always spawns on the shared, default executor. See spawn_on() to spawn
on a different named executor.
This utility exists so that it can be stubbed for testing without
interfering with the service spawns.
"""
return spawn_on(ExecutorType.DEFAULT, func, *args, **kwargs)
class StaticallyDelayingCancellableTaskExecutorWrapper:
"""Executor wrapper that submit work to another executor but delays each
task's submission with a statically defined delay and supports cancelling
the task during such delay.
Note that tasks that are actually started running in the real executor
might not be cancellable anymore depending on that executor and the
concurrency mode used. See
https://docs.python.org/3.12/library/concurrent.futures.html#concurrent.futures.Future.cancel
Note that shutting down the wrapper only shuts down its own scheduler
thread but does not shut down the real executor that is passed in __init__.
Note that this class does not support a different delay length for
different tasks.
"""
class Task:
def __init__(
self,
delay: float,
fn: ty.Callable[..., ty.Any],
args: tuple,
kwargs: dict
):
self.deadline = time.monotonic() + delay
self.future = futurist.Future()
self.fn = fn
self.args = args
self.kwargs = kwargs
@property
def remaining_delay(self):
return self.deadline - time.monotonic()
def __str__(self):
return (
f"Task(fn={self.fn}, "
f"remaining_delay={self.remaining_delay} "
f"future={self.future})")
def __init__(self, delay: float, executor: Executor):
"""Initialize the wrapper
:param delay: delay length in seconds
:param executor: executor object to run each task. It supports both
native threading and eventlet based executors.
"""
self._queue: queue.Queue = queue.Queue()
self._executor = executor
self._delay = delay
self._shutdown = threading.Condition()
self._shutdown_requested = False
self._sentinel = self.Task(0, lambda: None, (), {})
# We are intentionally not running our _run() in the executor
# as we cannot assume that the executor has more than one worker
# and our logic never finishes so it would consume one worker
# constantly.
self._thread = threading.Thread(target=self._run)
self._thread.daemon = True
self._thread.start()
@staticmethod
def _log(msg, *args):
LOG.debug(msg, *args)
@staticmethod
def _task_wrapper(task) -> None:
"""This wraps the original task so when it finishes in the real
executor the result of the task can be copied to the Future object
already returned to our caller from submit_with_delay(). So
the caller can get the result or exception from the task.
"""
try:
task.future.set_result(task.fn(*task.args, **task.kwargs))
except BaseException as e:
task.future.set_exception(e)
def _wait_for_deadline_then_set_running(self, task) -> bool:
"""Waiting for the task's deadline then mark it running
Wait can be interrupted by shutdown of the wrapper in such a case
the task's state is checked. If the task is cancelled then return
immediately with False. If the task is not cancelled then wait for
its deadline and eventually return True if the task is still not
cancelled.
If True is returned the future in the task is also atomically set to
running state and the future cannot be cancelled anymore.
"""
if task.remaining_delay <= 0:
return task.future.set_running_or_notify_cancel()
self._log("Waitig for the deadline of %s", task)
with self._shutdown:
shutdown = self._shutdown.wait_for(
lambda: self._shutdown_requested, task.remaining_delay)
if shutdown:
self._log(
"Shutdown is requested while waiting "
"for the deadline of %s", task)
if task.future.cancelled():
return False
self._log(
"%s is not cancelled so still waiting for its "
"deadline", task)
if task.remaining_delay > 0:
# Blocking here is fine as we have the assumption that
# no new task can arrive that has a deadline that is sooner
# than the deadline of the oldest task in the queue due to
# our static delay (and we assume that time travel is not
# allowed).
time.sleep(task.remaining_delay)
return task.future.set_running_or_notify_cancel()
def _run(self):
while True:
self._log("Waiting for the next task")
task: StaticallyDelayingCancellableTaskExecutorWrapper.Task = (
self._queue.get())
self._log("Received %s", task)
if task is self._sentinel:
# We are asked to terminate so exit the loop.
self._queue.task_done()
self._log("Sentinel received, thread is exiting")
return
if task.future.cancelled():
# The task was cancelled while it was in the queue.
# We don't need to run it. Just move on to the next task
self._log("%s was cancelled while queued, skipping", task)
self._queue.task_done()
continue
# The task is still valid, wait for its deadline
run_it = self._wait_for_deadline_then_set_running(task)
if not run_it:
# The task was cancelled during the delay period.
# We don't need to run it. Just move on to the next task.
self._log(
"%s was cancelled during its delay period, skipping", task)
self._queue.task_done()
continue
# Push the task to the real executor. We don't need to wait
# for the result here as the client can do that
# via the task.future we already returned from submit_with_delay()
try:
self._executor.submit(self._task_wrapper, task)
except BaseException as e:
# If for any reason we cannot submit a task then we should not
# let the exception escape as that will prevent our thread
# to terminate cleanly. Instead, we log and propagate back.
LOG.exception(
"Failed to submit %s to executor %s", task, self._executor)
task.future.set_exception(e)
self._queue.task_done()
continue
self._log("%s submitted to %s", task, self._executor)
# The task is not done from the executor perspective, but it is
# done from the _run() logic perspective. Signal it, so shutdown()
# can use Queue.join()
self._queue.task_done()
def submit_with_delay(
self, fn: ty.Callable[..., ty.Any], *args: ty.Any, **kwargs: ty.Any
) -> futurist.Future:
"""Submit work with delay."""
# We need this wide locking as we don't want to queue a task behind
# the sentinel as that task will never be processed.
with self._shutdown:
if self._shutdown_requested:
raise RuntimeError(
"Cannot schedule new tasks after being shutdown")
task = self.Task(self._delay, fn, args, kwargs)
self._queue.put(task)
self._log("Queued %s", task)
return task.future
def shutdown(self, wait: bool = True):
"""Shutdown the executor"""
with self._shutdown:
if not self._shutdown_requested:
# Ensure that our thread wakes at least one more time to allow
# it to exit by queuing up a sentinel task after the shutdown
# condition is set. This task won't be executed.
self._queue.put(self._sentinel)
self._log("Sentinel is queued")
self._shutdown_requested = True
self._shutdown.notify_all()
self._log("Shutdown is set")
# If wait is set we need to wait for our sentinel to be processed and
# therefore our thread to exit.
# NOTE(gibi): We are intentionally not shutting down the real executor
# as we are not the one created it or owning it so it might be shared
# between different callers.
if wait:
self._queue.join()
self._log("Queue joined")
self._thread.join()
self._log("Scheduler thread joined")
@property
def is_alive(self) -> bool:
return self._thread.is_alive()
-495
View File
@@ -21,9 +21,7 @@ import datetime
import functools
import hashlib
import inspect
import multiprocessing
import os
import queue
import random
import re
import shutil
@@ -32,7 +30,6 @@ import threading
import time
import typing as ty
import futurist
from keystoneauth1 import loading as ks_loading
import netaddr
from openstack import connection
@@ -41,10 +38,8 @@ import os_resource_classes as orc
from os_service_types import service_types
from oslo_concurrency import lockutils
from oslo_concurrency import processutils
from oslo_context import context as common_context
from oslo_log import log as logging
from oslo_utils import encodeutils
from oslo_utils import importutils
from oslo_utils import strutils
from oslo_utils import timeutils
@@ -53,9 +48,6 @@ from nova import exception
from nova.i18n import _
from nova import safe_utils
profiler = importutils.try_import('osprofiler.profiler')
CONF = nova.conf.CONF
LOG = logging.getLogger(__name__)
@@ -81,13 +73,6 @@ _FILE_CACHE: dict[str, dict] = {}
_SERVICE_TYPES = service_types.ServiceTypes()
# NOTE(gibi): futurist does not expose the common based class.
# NOTE(gibi): we can simplify this when eventlet is removed.
Executor: ty.TypeAlias = (
futurist.GreenThreadPoolExecutor | futurist.ThreadPoolExecutor)
DEFAULT_EXECUTOR: Executor | None = None
def cooperative_yield():
# TODO(ksambor) Remove all cooperative_yield calls after dropping Eventlet
@@ -104,48 +89,6 @@ def get_eventlet():
return eventlet
def destroy_default_executor():
"""Closes the executor and resets the global to None to allow forked worker
processes to properly init it.
"""
global DEFAULT_EXECUTOR
if DEFAULT_EXECUTOR:
LOG.info(
"The default thread pool %s is shutting down",
DEFAULT_EXECUTOR.name)
DEFAULT_EXECUTOR.shutdown()
LOG.info(
"The default thread pool %s is closed", DEFAULT_EXECUTOR.name)
DEFAULT_EXECUTOR = None
def create_executor(max_workers) -> Executor:
if concurrency_mode_threading():
executor = futurist.ThreadPoolExecutor(max_workers)
else:
executor = futurist.GreenThreadPoolExecutor(max_workers)
return executor
def _get_default_executor() -> Executor:
global DEFAULT_EXECUTOR
if not DEFAULT_EXECUTOR:
max_workers = (
CONF.default_thread_pool_size if concurrency_mode_threading()
else CONF.default_green_pool_size
)
DEFAULT_EXECUTOR = create_executor(max_workers)
pname = multiprocessing.current_process().name
executor_name = f"{pname}.default"
DEFAULT_EXECUTOR.name = executor_name
LOG.info("The default thread pool %s is initialized", executor_name)
return DEFAULT_EXECUTOR
# NOTE(mikal): this seems to have to stay for now to handle os-brick
# requirements. This makes me a sad panda.
def get_root_helper():
@@ -562,85 +505,6 @@ def validate_integer(value, name, min_value=None, max_value=None):
raise exception.InvalidInput(reason=str(e))
def _serialize_profile_info():
if not profiler:
return None
prof = profiler.get()
trace_info = None
if prof:
# FIXME(DinaBelova): we'll add profiler.get_info() method
# to extract this info -> we'll need to update these lines
trace_info = {
"hmac_key": prof.hmac_key,
"base_id": prof.get_base_id(),
"parent_id": prof.get_id()
}
return trace_info
def spawn(
func: ty.Callable[..., ty.Any], *args: ty.Any, **kwargs: ty.Any
) -> futurist.Future:
"""Passthrough method for eventlet.spawn.
This utility exists so that it can be stubbed for testing without
interfering with the service spawns.
It will also grab the context from the threadlocal store and add it to
the store on the new thread. This allows for continuity in logging the
context when using this method to spawn a new thread.
"""
return spawn_on(_get_default_executor(), func, *args, **kwargs)
def _executor_is_full(executor: Executor) -> bool:
if concurrency_mode_threading():
# TODO(gibi): Move this whole logic to futurist ThreadPoolExecutor
# so that we can avoid accessing the internals of the executor
with executor._shutdown_lock:
idle_workers = len([w for w in executor._workers if w.idle]) > 0
queued_tasks = executor._work_queue.qsize() > 0
return queued_tasks and not idle_workers
return False
def spawn_on(
executor: Executor,
func: ty.Callable[..., ty.Any],
*args: ty.Any, **kwargs: ty.Any,
) -> futurist.Future:
"""Passthrough method to run func on a thread in a given executor.
It will also grab the context from the threadlocal store and add it to
the store on the new thread. This allows for continuity in logging the
context when using this method to spawn a new thread.
"""
_log_executor_stats(executor)
if _executor_is_full(executor):
LOG.warning(
"The %s pool does not have free threads so the task %s will be "
"queued. If this happens repeatedly then the size of the pool is "
"too small for the load or there are stuck threads filling the "
"pool.", getattr(executor, "name", "unknown"), func)
_context = common_context.get_current()
profiler_info = _serialize_profile_info()
@functools.wraps(func)
def context_wrapper(*args, **kwargs):
# NOTE: If update_store is not called after spawning a thread, it won't
# be available for the logger to pull from threadlocal storage.
if _context is not None:
_context.update_store()
if profiler_info and profiler:
profiler.init(**profiler_info)
return func(*args, **kwargs)
return executor.submit(context_wrapper, *args, **kwargs)
def is_none_string(val):
"""Check if a string represents a None value.
"""
@@ -1318,117 +1182,6 @@ def concurrency_mode_threading():
return not monkey_patch.is_patched()
SCATTER_GATHER_EXECUTOR: Executor | None = None
def get_scatter_gather_executor() -> Executor:
"""Returns the executor used for scatter/gather operations."""
global SCATTER_GATHER_EXECUTOR
if not SCATTER_GATHER_EXECUTOR:
max_workers = (
CONF.cell_worker_thread_pool_size
if concurrency_mode_threading() else 1000
)
SCATTER_GATHER_EXECUTOR = create_executor(max_workers)
pname = multiprocessing.current_process().name
executor_name = f"{pname}.cell_worker"
SCATTER_GATHER_EXECUTOR.name = executor_name
LOG.info("The cell worker thread pool %s is initialized",
executor_name)
return SCATTER_GATHER_EXECUTOR
def destroy_scatter_gather_executor():
"""Closes the executor and resets the global to None to allow forked worker
processes to properly init it.
"""
global SCATTER_GATHER_EXECUTOR
if SCATTER_GATHER_EXECUTOR:
LOG.info(
"The cell worker thread pool %s is shutting down",
SCATTER_GATHER_EXECUTOR.name)
SCATTER_GATHER_EXECUTOR.shutdown()
LOG.info(
"The cell worker thread pool %s is closed",
SCATTER_GATHER_EXECUTOR.name)
SCATTER_GATHER_EXECUTOR = None
CACHE_IMAGES_EXECUTOR: Executor | None = None
def get_cache_images_executor() -> Executor:
"""Returns the executor used for cache images operations."""
global CACHE_IMAGES_EXECUTOR
if not CACHE_IMAGES_EXECUTOR:
max_workers = CONF.image_cache.precache_concurrency
CACHE_IMAGES_EXECUTOR = create_executor(max_workers)
pname = multiprocessing.current_process().name
executor_name = f"{pname}.cache_images"
CACHE_IMAGES_EXECUTOR.name = executor_name
LOG.info("The cache images executor %s is initialized", executor_name)
return CACHE_IMAGES_EXECUTOR
def destroy_cache_images_executor():
"""Closes the executor and resets the global to None to allow forked worker
processes to properly init it.
"""
global CACHE_IMAGES_EXECUTOR
if CACHE_IMAGES_EXECUTOR:
LOG.info(
"The cache images thread pool %s is shutting down",
CACHE_IMAGES_EXECUTOR.name)
CACHE_IMAGES_EXECUTOR.shutdown()
LOG.info(
"The cache images thread pool %s is closed",
CACHE_IMAGES_EXECUTOR.name)
CACHE_IMAGES_EXECUTOR = None
def _log_executor_stats(executor: Executor) -> None:
if CONF.thread_pool_statistic_period < 0:
return
last_stats = getattr(executor, "last_stats", None)
name = getattr(executor, "name", "unknown")
allowed_stat_age = time.monotonic() - CONF.thread_pool_statistic_period
if last_stats and last_stats > allowed_stat_age:
return
executor.last_stats = time.monotonic()
stats: futurist.ExecutorStatistics = executor.statistics
if isinstance(executor, futurist.ThreadPoolExecutor):
LOG.debug(
"State of %s ThreadPoolExecutor when submitting a new task: "
"max_workers: %d, workers: %d, idle workers: %d, queued work: %d, "
"stats: %s",
name,
executor._max_workers, len(executor._workers),
len([w for w in executor._workers if w.idle]),
executor._work_queue.qsize(), stats)
elif isinstance(executor, futurist.GreenThreadPoolExecutor):
LOG.debug(
"State of %s GreenThreadPoolExecutor when submitting a new task: "
"workers: %d, max_workers: %d, work queued length: %d, stats: %s",
name,
len(executor._pool.coroutines_running), executor._pool.size,
executor._delayed_work.unfinished_tasks, stats)
def tpool_wrap(target, autowrap=()):
"""Wrap the target into an eventlet Tpool Proxy object if running
in eventlet mode. In threading mode no wrapping is applied.
@@ -1438,251 +1191,3 @@ def tpool_wrap(target, autowrap=()):
else:
return get_eventlet().tpool.Proxy(target, autowrap=autowrap)
class StaticallyDelayingCancellableTaskExecutorWrapper:
"""Executor wrapper that submit work to another executor but delays each
task's submission with a statically defined delay and supports cancelling
the task during such delay.
Note that tasks that are actually started running in the real executor
might not be cancellable anymore depending on that executor and the
concurrency mode used. See
https://docs.python.org/3.12/library/concurrent.futures.html#concurrent.futures.Future.cancel
Note that shutting down the wrapper only shuts down its own scheduler
thread but does not shut down the real executor that is passed in __init__.
Note that this class does not support a different delay length for
different tasks.
"""
class Task:
def __init__(
self,
delay: float,
fn: ty.Callable[..., ty.Any],
args: tuple,
kwargs: dict
):
self.deadline = time.monotonic() + delay
self.future = futurist.Future()
self.fn = fn
self.args = args
self.kwargs = kwargs
@property
def remaining_delay(self):
return self.deadline - time.monotonic()
def __str__(self):
return (
f"Task(fn={self.fn}, "
f"remaining_delay={self.remaining_delay} "
f"future={self.future})")
def __init__(self, delay: float, executor: Executor):
"""Initialize the wrapper
:param delay: delay length in seconds
:param executor: executor object to run each task. It supports both
native threading and eventlet based executors.
"""
self._queue: queue.Queue = queue.Queue()
self._executor = executor
self._delay = delay
self._shutdown = threading.Condition()
self._shutdown_requested = False
self._sentinel = self.Task(0, lambda: None, (), {})
# We are intentionally not running our _run() in the executor
# as we cannot assume that the executor has more than one worker
# and our logic never finishes so it would consume one worker
# constantly.
self._thread = threading.Thread(target=self._run)
self._thread.daemon = True
self._thread.start()
@staticmethod
def _log(msg, *args):
LOG.debug(msg, *args)
@staticmethod
def _task_wrapper(task) -> None:
"""This wraps the original task so when it finishes in the real
executor the result of the task can be copied to the Future object
already returned to our caller from submit_with_delay(). So
the caller can get the result or exception from the task.
"""
try:
task.future.set_result(task.fn(*task.args, **task.kwargs))
except BaseException as e:
task.future.set_exception(e)
def _wait_for_deadline_then_set_running(self, task) -> bool:
"""Waiting for the task's deadline then mark it running
Wait can be interrupted by shutdown of the wrapper in such a case
the task's state is checked. If the task is cancelled then return
immediately with False. If the task is not cancelled then wait for
its deadline and eventually return True if the task is still not
cancelled.
If True is returned the future in the task is also atomically set to
running state and the future cannot be cancelled anymore.
"""
if task.remaining_delay <= 0:
return task.future.set_running_or_notify_cancel()
self._log("Waitig for the deadline of %s", task)
with self._shutdown:
shutdown = self._shutdown.wait_for(
lambda: self._shutdown_requested, task.remaining_delay)
if shutdown:
self._log(
"Shutdown is requested while waiting "
"for the deadline of %s", task)
if task.future.cancelled():
return False
self._log(
"%s is not cancelled so still waiting for its "
"deadline", task)
if task.remaining_delay > 0:
# Blocking here is fine as we have the assumption that
# no new task can arrive that has a deadline that is sooner
# than the deadline of the oldest task in the queue due to
# our static delay (and we assume that time travel is not
# allowed).
time.sleep(task.remaining_delay)
return task.future.set_running_or_notify_cancel()
def _run(self):
while True:
self._log("Waiting for the next task")
task: StaticallyDelayingCancellableTaskExecutorWrapper.Task = (
self._queue.get())
self._log("Received %s", task)
if task is self._sentinel:
# We are asked to terminate so exit the loop.
self._queue.task_done()
self._log("Sentinel received, thread is exiting")
return
if task.future.cancelled():
# The task was cancelled while it was in the queue.
# We don't need to run it. Just move on to the next task
self._log("%s was cancelled while queued, skipping", task)
self._queue.task_done()
continue
# The task is still valid, wait for its deadline
run_it = self._wait_for_deadline_then_set_running(task)
if not run_it:
# The task was cancelled during the delay period.
# We don't need to run it. Just move on to the next task.
self._log(
"%s was cancelled during its delay period, skipping", task)
self._queue.task_done()
continue
# Push the task to the real executor. We don't need to wait
# for the result here as the client can do that
# via the task.future we already returned from submit_with_delay()
try:
self._executor.submit(self._task_wrapper, task)
except BaseException as e:
# If for any reason we cannot submit a task then we should not
# let the exception escape as that will prevent our thread
# to terminate cleanly. Instead, we log and propagate back.
LOG.exception(
"Failed to submit %s to executor %s", task, self._executor)
task.future.set_exception(e)
self._queue.task_done()
continue
self._log("%s submitted to %s", task, self._executor)
# The task is not done from the executor perspective, but it is
# done from the _run() logic perspective. Signal it, so shutdown()
# can use Queue.join()
self._queue.task_done()
def submit_with_delay(
self, fn: ty.Callable[..., ty.Any], *args: ty.Any, **kwargs: ty.Any
) -> futurist.Future:
"""Submit work with delay."""
# We need this wide locking as we don't want to queue a task behind
# the sentinel as that task will never be processed.
with self._shutdown:
if self._shutdown_requested:
raise RuntimeError(
"Cannot schedule new tasks after being shutdown")
task = self.Task(self._delay, fn, args, kwargs)
self._queue.put(task)
self._log("Queued %s", task)
return task.future
def shutdown(self, wait: bool = True):
"""Shutdown the executor"""
with self._shutdown:
if not self._shutdown_requested:
# Ensure that our thread wakes at least one more time to allow
# it to exit by queuing up a sentinel task after the shutdown
# condition is set. This task won't be executed.
self._queue.put(self._sentinel)
self._log("Sentinel is queued")
self._shutdown_requested = True
self._shutdown.notify_all()
self._log("Shutdown is set")
# If wait is set we need to wait for our sentinel to be processed and
# therefore our thread to exit.
# NOTE(gibi): We are intentionally not shutting down the real executor
# as we are not the one created it or owning it so it might be shared
# between different callers.
if wait:
self._queue.join()
self._log("Queue joined")
self._thread.join()
self._log("Scheduler thread joined")
@property
def is_alive(self) -> bool:
return self._thread.is_alive()
LONG_TASK_EXECUTOR: Executor | None = None
def get_long_task_executor(max_workers) -> Executor:
"""Returns the executor used for long compute operations."""
global LONG_TASK_EXECUTOR
if not LONG_TASK_EXECUTOR:
LONG_TASK_EXECUTOR = create_executor(max_workers)
pname = multiprocessing.current_process().name
executor_name = f"{pname}.long_task"
LONG_TASK_EXECUTOR.name = executor_name
LOG.info("The long task thread pool %s is initialized",
executor_name)
return LONG_TASK_EXECUTOR
def destroy_long_task_executor():
"""Closes the executor and resets the global to None"""
global LONG_TASK_EXECUTOR
if LONG_TASK_EXECUTOR:
LOG.info(
"The thread pool %s is shutting down", LONG_TASK_EXECUTOR.name)
LONG_TASK_EXECUTOR.shutdown()
LOG.info("The thread pool %s is closed", LONG_TASK_EXECUTOR.name)
LONG_TASK_EXECUTOR = None
+3 -1
View File
@@ -99,6 +99,7 @@ import nova.privsep.libvirt
import nova.privsep.path
import nova.privsep.utils
from nova.storage import rbd_utils
from nova import thread_pool_factory
from nova import utils
from nova import version
from nova.virt import block_device as driver_block_device
@@ -11824,7 +11825,8 @@ class LibvirtDriver(driver.ComputeDriver):
disk_paths, device_names = self._live_migration_copy_disk_paths(
context, instance, guest)
future = utils.spawn(self._live_migration_operation,
future = thread_pool_factory.spawn(
self._live_migration_operation,
context, instance, dest,
block_migration,
migrate_data, guest,
+6 -4
View File
@@ -53,6 +53,7 @@ from nova.i18n import _
from nova.objects import fields
from nova.pci import utils as pci_utils
from nova import rpc
from nova import thread_pool_factory
from nova import utils
from nova.virt import event as virtevent
from nova.virt.libvirt import config as vconfig
@@ -144,7 +145,7 @@ class _EventletLibvirtEventHandler(LibvirtEventHandler):
self._event_thread.start()
LOG.debug("Starting event dispatch greenthread")
utils.spawn(self._dispatch_thread)
thread_pool_factory.spawn(self._dispatch_thread)
def _queue_event(self, event):
"""Puts an event on the queue for dispatch.
@@ -542,15 +543,16 @@ class Host(object):
"""Initializes the libvirt events subsystem.
"""
self._delayed_executor = (
utils.StaticallyDelayingCancellableTaskExecutorWrapper(
delay=15, executor=utils._get_default_executor()))
thread_pool_factory.StaticallyDelayingCancellableTaskExecutorWrapper(
delay=15, executor=thread_pool_factory.get_executor(
thread_pool_factory.ExecutorType.DEFAULT)))
self._event_handler.start()
# This thread is just for async connection closed event handling.
# In eventlet mode it only handles tasks within the main thread with
# the eventlet hub.
LOG.debug("Starting connection event dispatch thread")
utils.spawn(self._conn_event_thread)
thread_pool_factory.spawn(self._conn_event_thread)
def _get_new_connection(self):
# call with _wrapped_conn_lock held