From 280007c870b671de9439efba71fd3916403d15da Mon Sep 17 00:00:00 2001 From: Sean Mooney Date: Tue, 28 Apr 2020 23:20:16 +0100 Subject: [PATCH] Silence amqp heartbeat warning When the nova api is executing under uWSGI or MOD_WSGI the lifetime of the amqp heartbeat thread is controlled by the wsgi server. As a result when the nova api is run in this configuration we expect that the heartbeat thread will be suspended and heartbeats will be missed when the wsgi server suspends execution of the wsgi application. This change adds a python logging filter to suppress the reporting of heartbeat warnings as this behavior is expected. Since the operator cannot do anything to address the issue the warning is just noise and many operators and customers find it to be off-putting. Changes: nova/config.py in python 3 log filters can simply be a function that returns true or false and accepts single argument which is the record to filter. To support python 2 the rabbit_heartbeat_filter has been converted into an instance of a new HeartbeatFilter class with its implementation moved to the filter method. Change-Id: I642b1e3ed6de2be4dcc19fe214f84095d2e1d31a Closes-Bug: #1825584 (cherry picked from commit d6f664524dfa4eecbe3e16fda0fec5359faf84c6) (cherry picked from commit 1caddc4250b4acbdb8a945c8f26a0792bee5e2ed) --- nova/config.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/nova/config.py b/nova/config.py index 4be8e342392c..865aac6ce8b5 100644 --- a/nova/config.py +++ b/nova/config.py @@ -15,6 +15,8 @@ # License for the specific language governing permissions and limitations # under the License. +import logging + from oslo_log import log from oslo_utils import importutils @@ -30,6 +32,24 @@ profiler = importutils.try_import('osprofiler.opts') CONF = nova.conf.CONF +class HeartbeatFilter(logging.Filter): + + def filter(self, record): + # Note the type in the log message was fixed in + # change Id11db4113c9b1c3add602192c1e915218704ef27 + # but we handle both form to allow this to be backported + # without consideration of the version of oslo.messaging used. + # TODO(sean-k-mooney): remove support for typo in follow up + # to allow this to be easily backported without modification. + messages = [ + "Unexpected error during heartbeart thread processing", + "Unexpected error during heartbeat thread processing"] + return not any(msg in record.msg for msg in messages) + + +rabbit_heartbeat_filter = HeartbeatFilter() + + def parse_args(argv, default_config_files=None, configure_db=True, init_rpc=True): log.register_options(CONF) @@ -40,6 +60,12 @@ def parse_args(argv, default_config_files=None, configure_db=True, else: extra_default_log_levels = ['glanceclient=WARN'] + # NOTE(sean-k-mooney): this filter addresses bug #1825584 + # https://bugs.launchpad.net/nova/+bug/1825584 + # eventlet monkey-patching breaks AMQP heartbeat on uWSGI + rabbit_logger = logging.getLogger('oslo.messaging._drivers.impl_rabbit') + rabbit_logger.addFilter(rabbit_heartbeat_filter) + # NOTE(danms): DEBUG logging in privsep will result in some large # and potentially sensitive things being logged. extra_default_log_levels.append('oslo.privsep.daemon=INFO')