From 46401ef6665d473b51a781a566a29b7b24eb711f Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Fri, 24 Sep 2021 14:20:43 +0000 Subject: [PATCH] api: enable oslo.reports when using uWSGI At the moment, oslo.reports is enabled when running nova-api standalone, but not when using uWSGI. We're now updating the uwsgi entry point as well to include the oslo.reports hook, which is extremely helpful when debugging deadlocks. Change-Id: I605f0e40417fe9b0a383cc8b3fefa1325f9690d9 --- doc/source/reference/gmr.rst | 31 ++++++++++++++++--- nova/api/openstack/wsgi_app.py | 10 ++++-- .../notes/uwsgi-gmr-c00631db79836340.yaml | 6 ++++ 3 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 nova/releasenotes/notes/uwsgi-gmr-c00631db79836340.yaml diff --git a/doc/source/reference/gmr.rst b/doc/source/reference/gmr.rst index 93250be45466..ceff7afb827d 100644 --- a/doc/source/reference/gmr.rst +++ b/doc/source/reference/gmr.rst @@ -23,7 +23,23 @@ Generating a GMR A *GMR* can be generated by sending the *USR2* signal to any Nova process with support (see below). The *GMR* will then be outputted standard error for that particular process. -For example, suppose that ``nova-api`` has process id ``8675``, and was run with ``2>/var/log/nova/nova-api-err.log``. Then, ``kill -USR2 8675`` will trigger the Guru Meditation report to be printed to ``/var/log/nova/nova-api-err.log``. +For example, suppose that ``nova-compute`` has process id ``8675``, and was run with ``2>/var/log/nova/nova-compute-err.log``. Then, ``kill -USR2 8675`` will trigger the Guru Meditation report to be printed to ``/var/log/nova/nova-compute-err.log``. + +Nova API is commonly run under uWSGI, which intercepts ``SIGUSR2`` signals. In this case, a file trigger may be used instead: + +.. code-block:: ini + + [oslo_reports] + log_dir = /var/log/nova + file_event_handler = /var/log/nova/gmr_trigger + +Whenever the trigger file is modified, a *GMR* will be generated. To get a +report, one may use ``touch /var/log/nova/gmr_trigger``. +Note that the configured file trigger must exist when Nova starts. + +If a log dir is specified, the report will be written to a file within that +directory instead of ``stderr``. The report file will be named +``${serviceName}_gurumeditation_${timestamp}``. Structure of a GMR ------------------ @@ -52,20 +68,27 @@ First import the module, as well as the Nova version module: .. code-block:: python from oslo_reports import guru_meditation_report as gmr + from oslo_reports import opts as gmr_opts from nova import version Then, register any additional sections (optional): .. code-block:: python - TextGuruMeditation.register_section('Some Special Section', - some_section_generator) + gmr.TextGuruMeditation.register_section('Some Special Section', + some_section_generator) Finally (under main), before running the "main loop" of the executable (usually ``service.server(server)`` or something similar), register the *GMR* hook: .. code-block:: python - TextGuruMeditation.setup_autorun(version) + gmr_opts.set_defaults(CONF) + gmr.TextGuruMeditation.setup_autorun( + version, conf=CONF, service_name=service_name) + +The service name is used when generating report files. If unspecified, *GMR* +tries to automatically detect the binary name using the stack trace but usually +ends up with ``thread.py``. Extending the GMR ----------------- diff --git a/nova/api/openstack/wsgi_app.py b/nova/api/openstack/wsgi_app.py index 947958a9e484..ff2bcae2151f 100644 --- a/nova/api/openstack/wsgi_app.py +++ b/nova/api/openstack/wsgi_app.py @@ -16,6 +16,8 @@ import sys from oslo_config import cfg from oslo_log import log as logging +from oslo_reports import guru_meditation_report as gmr +from oslo_reports import opts as gmr_opts from oslo_service import _options as service_opts from paste import deploy @@ -25,6 +27,7 @@ from nova import exception from nova import objects from nova import service from nova import utils +from nova import version CONF = cfg.CONF @@ -79,7 +82,7 @@ def error_application(exc, name): @utils.run_once('Global data already initialized, not re-initializing.', LOG.info) -def init_global_data(conf_files): +def init_global_data(conf_files, service_name): # NOTE(melwitt): parse_args initializes logging and calls global rpc.init() # and db_api.configure(). The db_api.configure() call does not initiate any # connection to the database. @@ -89,6 +92,9 @@ def init_global_data(conf_files): config.parse_args(sys.argv, default_config_files=conf_files) logging.setup(CONF, "nova") + gmr_opts.set_defaults(CONF) + gmr.TextGuruMeditation.setup_autorun( + version, conf=CONF, service_name=service_name) # dump conf at debug (log_options option comes from oslo.service) # FIXME(mriedem): This is gross but we don't have a public hook into @@ -110,7 +116,7 @@ def init_application(name): # apache/mod_wsgi reloads the init_application script. So, we initialize # global data separately and decorate the method to run only once in a # python interpreter instance. - init_global_data(conf_files) + init_global_data(conf_files, name) try: _setup_service(CONF.host, name) diff --git a/nova/releasenotes/notes/uwsgi-gmr-c00631db79836340.yaml b/nova/releasenotes/notes/uwsgi-gmr-c00631db79836340.yaml new file mode 100644 index 000000000000..d6b6e45968e6 --- /dev/null +++ b/nova/releasenotes/notes/uwsgi-gmr-c00631db79836340.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Guru Meditation Reports can now be generated for the Nova API service + when running under uWSGI. Note that uWSGI intercepts SIGUSR2 signals, + so a file trigger should be used instead.